public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 5/5] WIP unify handling of attributes and expressions
36+ messages / 8 participants
[nested] [flat]
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* [PATCH 5/5] WIP unify handling of attributes and expressions
@ 2021-03-04 03:53 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Tomas Vondra @ 2021-03-04 03:53 UTC (permalink / raw)
---
src/backend/statistics/dependencies.c | 82 ++------
src/backend/statistics/extended_stats.c | 180 +++++++++++-------
src/backend/statistics/mcv.c | 89 ++-------
src/backend/statistics/mvdistinct.c | 138 +++-----------
.../statistics/extended_stats_internal.h | 40 ++--
5 files changed, 183 insertions(+), 346 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index 602301b724..14d6503e1a 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -70,10 +70,7 @@ static void generate_dependencies(DependencyGenerator state);
static DependencyGenerator DependencyGenerator_init(int n, int k);
static void DependencyGenerator_free(DependencyGenerator state);
static AttrNumber *DependencyGenerator_next(DependencyGenerator state);
-static double dependency_degree(int numrows, HeapTuple *rows,
- ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs);
+static double dependency_degree(StatBuildData *data, int k, AttrNumber *dependency);
static bool dependency_is_fully_matched(MVDependency *dependency,
Bitmapset *attnums);
static bool dependency_is_compatible_clause(Node *clause, Index relid,
@@ -222,17 +219,13 @@ DependencyGenerator_next(DependencyGenerator state)
* the last one.
*/
static double
-dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
- AttrNumber *dependency, VacAttrStats **stats,
- Bitmapset *attrs)
+dependency_degree(StatBuildData *data, int k, AttrNumber *dependency)
{
int i,
nitems;
MultiSortSupport mss;
SortItem *items;
- AttrNumber *attnums;
AttrNumber *attnums_dep;
- int numattrs;
/* counters valid within a group */
int group_size = 0;
@@ -247,16 +240,9 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* sort info for all attributes columns */
mss = multi_sort_init(k);
- /*
- * Transform the attrs from bitmap to an array to make accessing the i-th
- * member easier, and then construct a filtered version with only attnums
- * referenced by the dependency we validate.
- */
- attnums = build_attnums_array(attrs, exprs->nexprs, &numattrs);
-
attnums_dep = (AttrNumber *) palloc(k * sizeof(AttrNumber));
for (i = 0; i < k; i++)
- attnums_dep[i] = attnums[dependency[i]];
+ attnums_dep[i] = data->attnums[dependency[i]];
/*
* Verify the dependency (a,b,...)->z, using a rather simple algorithm:
@@ -274,7 +260,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
/* prepare the sort function for the dimensions */
for (i = 0; i < k; i++)
{
- VacAttrStats *colstat = stats[dependency[i]];
+ VacAttrStats *colstat = data->stats[dependency[i]];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -293,8 +279,7 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* descriptor. For now that assumption holds, but it might change in the
* future for example if we support statistics on multiple tables.
*/
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, k, attnums_dep);
+ items = build_sorted_items(data, &nitems, mss, k, attnums_dep);
/*
* Walk through the sorted array, split it into rows according to the
@@ -340,11 +325,10 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
pfree(items);
pfree(mss);
- pfree(attnums);
pfree(attnums_dep);
/* Compute the 'degree of validity' as (supporting/total). */
- return (n_supporting_rows * 1.0 / numrows);
+ return (n_supporting_rows * 1.0 / data->numrows);
}
/*
@@ -364,54 +348,15 @@ dependency_degree(int numrows, HeapTuple *rows, ExprInfo *exprs, int k,
* (c) -> b
*/
MVDependencies *
-statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_dependencies_build(StatBuildData *data)
{
int i,
k;
- int numattrs;
- AttrNumber *attnums;
- int nattnums;
/* result */
MVDependencies *dependencies = NULL;
- /*
- * Transform the bms into an array of attnums, to make accessing i-th
- * member easier. We add the expressions first, represented by negative
- * attnums (this is OK, we don't allow statistics on system attributes),
- * and then regular attributes.
- */
- nattnums = bms_num_members(attrs) + exprs->nexprs;
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * nattnums);
-
- numattrs = 0;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- /*
- * and then regular attributes
- *
- * XXX Maybe add this in the opposite order, just like in MCV? first
- * regular attnums, then exressions.
- */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- Assert(numattrs >= 2);
- Assert(numattrs == nattnums);
-
- /*
- * Build a new bitmapset of attnums, offset by number of expressions (this
- * is needed, because bitmaps can store only non-negative values).
- */
- attrs = NULL;
- for (i = 0; i < numattrs; i++)
- attrs = bms_add_member(attrs, attnums[i] + exprs->nexprs);
+ Assert(data->nattnums >= 2);
/*
* We'll try build functional dependencies starting from the smallest ones
@@ -419,12 +364,12 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
* included in the statistics object. We start from the smallest ones
* because we want to be able to skip already implied ones.
*/
- for (k = 2; k <= numattrs; k++)
+ for (k = 2; k <= data->nattnums; k++)
{
AttrNumber *dependency; /* array with k elements */
/* prepare a DependencyGenerator of variation */
- DependencyGenerator DependencyGenerator = DependencyGenerator_init(numattrs, k);
+ DependencyGenerator DependencyGenerator = DependencyGenerator_init(data->nattnums, k);
/* generate all possible variations of k values (out of n) */
while ((dependency = DependencyGenerator_next(DependencyGenerator)))
@@ -433,8 +378,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
MVDependency *d;
/* compute how valid the dependency seems */
- degree = dependency_degree(numrows, rows, exprs, k, dependency,
- stats, attrs);
+ degree = dependency_degree(data, k, dependency);
/*
* if the dependency seems entirely invalid, don't store it
@@ -449,7 +393,7 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
d->degree = degree;
d->nattributes = k;
for (i = 0; i < k; i++)
- d->attributes[i] = attnums[dependency[i]];
+ d->attributes[i] = data->attnums[dependency[i]];
/* initialize the list of dependencies */
if (dependencies == NULL)
@@ -477,8 +421,6 @@ statext_dependencies_build(int numrows, HeapTuple *rows,
DependencyGenerator_free(DependencyGenerator);
}
- pfree(attrs);
-
return dependencies;
}
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 95b2cc683e..5b3fa523e9 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -96,8 +96,11 @@ static Datum serialize_expr_stats(AnlExprData *exprdata, int nexprs);
static Datum expr_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
static AnlExprData *build_expr_data(List *exprs);
static VacAttrStats *examine_expression(Node *expr);
-static ExprInfo *evaluate_expressions(Relation rel, List *exprs,
- int numrows, HeapTuple *rows);
+
+static StatBuildData *make_build_data(Relation onerel, StatExtEntry *stat,
+ int numrows, HeapTuple *rows,
+ VacAttrStats **stats);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -156,7 +159,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
VacAttrStats **stats;
ListCell *lc2;
int stattarget;
- ExprInfo *exprs;
+ StatBuildData *data;
/*
* Check if we can build these stats based on the column analyzed. If
@@ -191,7 +194,7 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
continue;
/* evaluate expressions (if the statistics has any) */
- exprs = evaluate_expressions(onerel, stat->exprs, numrows, rows);
+ data = make_build_data(onerel, stat, numrows, rows, stats);
/* compute statistic of each requested type */
foreach(lc2, stat->types)
@@ -199,16 +202,11 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
char t = (char) lfirst_int(lc2);
if (t == STATS_EXT_NDISTINCT)
- ndistinct = statext_ndistinct_build(totalrows, numrows, rows,
- exprs, stat->columns,
- stats);
+ ndistinct = statext_ndistinct_build(totalrows, data);
else if (t == STATS_EXT_DEPENDENCIES)
- dependencies = statext_dependencies_build(numrows, rows,
- exprs, stat->columns,
- stats);
+ dependencies = statext_dependencies_build(data);
else if (t == STATS_EXT_MCV)
- mcv = statext_mcv_build(numrows, rows, exprs, stat->columns,
- stats, totalrows, stattarget);
+ mcv = statext_mcv_build(data, totalrows, stattarget);
else if (t == STATS_EXT_EXPRESSIONS)
{
AnlExprData *exprdata;
@@ -236,7 +234,8 @@ BuildRelationExtStatistics(Relation onerel, double totalrows,
pgstat_progress_update_param(PROGRESS_ANALYZE_EXT_STATS_COMPUTED,
++ext_cnt);
- pfree(exprs);
+ /* free the build data (allocated as a single chunk) */
+ pfree(data);
}
table_close(pg_stext, RowExclusiveLock);
@@ -937,30 +936,31 @@ build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs)
* can simply pfree the return value to release all of it.
*/
SortItem *
-build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
- TupleDesc tdesc, MultiSortSupport mss,
+build_sorted_items(StatBuildData *data, int *nitems,
+ MultiSortSupport mss,
int numattrs, AttrNumber *attnums)
{
int i,
j,
len,
- idx;
- int nvalues = numrows * numattrs;
+ nrows;
+ int nvalues = data->numrows * numattrs;
SortItem *items;
Datum *values;
bool *isnull;
char *ptr;
+ int *typlen;
/* Compute the total amount of memory we need (both items and values). */
- len = numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
+ len = data->numrows * sizeof(SortItem) + nvalues * (sizeof(Datum) + sizeof(bool));
/* Allocate the memory and split it into the pieces. */
ptr = palloc0(len);
/* items to sort */
items = (SortItem *) ptr;
- ptr += numrows * sizeof(SortItem);
+ ptr += data->numrows * sizeof(SortItem);
/* values and null flags */
values = (Datum *) ptr;
@@ -973,13 +973,24 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
Assert((ptr - (char *) items) == len);
/* fix the pointers to Datum and bool arrays */
- idx = 0;
- for (i = 0; i < numrows; i++)
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
{
- bool toowide = false;
+ items[nrows].values = &values[nrows * numattrs];
+ items[nrows].isnull = &isnull[nrows * numattrs];
+
+ nrows++;
+ }
+
+ /* build a local cache of typlen for all attributes */
+ typlen = (int *) palloc(sizeof(int) * data->nattnums);
+ for (i = 0; i < data->nattnums; i++)
+ typlen[i] = get_typlen(data->stats[i]->attrtypid);
- items[idx].values = &values[idx * numattrs];
- items[idx].isnull = &isnull[idx * numattrs];
+ nrows = 0;
+ for (i = 0; i < data->numrows; i++)
+ {
+ bool toowide = false;
/* load the values/null flags from sample rows */
for (j = 0; j < numattrs; j++)
@@ -989,22 +1000,20 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
int attlen;
AttrNumber attnum = attnums[j];
- if (AttrNumberIsForUserDefinedAttr(attnum))
+ int idx;
+
+ /* match attnum to the pre-calculated data */
+ for (idx = 0; idx < data->nattnums; idx++)
{
- value = heap_getattr(rows[i], attnum, tdesc, &isnull);
- attlen = TupleDescAttr(tdesc, attnum - 1)->attlen;
+ if (attnum == data->attnums[idx])
+ break;
}
- else
- {
- int idx = -(attnums[j] + 1);
-
- Assert((idx >= 0) && (idx < exprs->nexprs));
- value = exprs->values[idx][i];
- isnull = exprs->nulls[idx][i];
+ Assert(idx < data->nattnums);
- attlen = get_typlen(exprs->types[idx]);
- }
+ value = data->values[idx][i];
+ isnull = data->nulls[idx][i];
+ attlen = typlen[idx];
/*
* If this is a varlena value, check if it's too wide and if yes
@@ -1026,21 +1035,21 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
value = PointerGetDatum(PG_DETOAST_DATUM(value));
}
- items[idx].values[j] = value;
- items[idx].isnull[j] = isnull;
+ items[nrows].values[j] = value;
+ items[nrows].isnull[j] = isnull;
}
if (toowide)
continue;
- idx++;
+ nrows++;
}
/* store the actual number of items (ignoring the too-wide ones) */
- *nitems = idx;
+ *nitems = nrows;
/* all items were too wide */
- if (idx == 0)
+ if (nrows == 0)
{
/* everything is allocated as a single chunk */
pfree(items);
@@ -1048,7 +1057,7 @@ build_sorted_items(int numrows, int *nitems, HeapTuple *rows, ExprInfo *exprs,
}
/* do the sort, using the multi-sort */
- qsort_arg((void *) items, idx, sizeof(SortItem),
+ qsort_arg((void *) items, nrows, sizeof(SortItem),
multi_sort_compare, mss);
return items;
@@ -2434,59 +2443,61 @@ statext_expressions_load(Oid stxoid, int idx)
* all the requested statistics types. This matters especially for
* expensive expressions, of course.
*/
-static ExprInfo *
-evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
+static StatBuildData *
+make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
+ VacAttrStats **stats)
{
/* evaluated expressions */
- ExprInfo *result;
+ StatBuildData *result;
char *ptr;
Size len;
int i;
+ int k;
int idx;
TupleTableSlot *slot;
EState *estate;
ExprContext *econtext;
List *exprstates = NIL;
- int nexprs = list_length(exprs);
+ int nkeys = bms_num_members(stat->columns) + list_length(stat->exprs);
ListCell *lc;
/* allocate everything as a single chunk, so we can free it easily */
- len = MAXALIGN(sizeof(ExprInfo));
- len += MAXALIGN(sizeof(Oid) * nexprs); /* types */
- len += MAXALIGN(sizeof(Oid) * nexprs); /* collations */
+ len = MAXALIGN(sizeof(StatBuildData));
+ len += MAXALIGN(sizeof(AttrNumber) * nkeys); /* attnums */
+ len += MAXALIGN(sizeof(VacAttrStats *) * nkeys); /* stats */
/* values */
- len += MAXALIGN(sizeof(Datum *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(Datum) * numrows);
+ len += MAXALIGN(sizeof(Datum *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(Datum) * numrows);
/* nulls */
- len += MAXALIGN(sizeof(bool *) * nexprs);
- len += nexprs * MAXALIGN(sizeof(bool) * numrows);
+ len += MAXALIGN(sizeof(bool *) * nkeys);
+ len += nkeys * MAXALIGN(sizeof(bool) * numrows);
ptr = palloc(len);
/* set the pointers */
- result = (ExprInfo *) ptr;
- ptr += MAXALIGN(sizeof(ExprInfo));
+ result = (StatBuildData *) ptr;
+ ptr += MAXALIGN(sizeof(StatBuildData));
- /* types */
- result->types = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* attnums */
+ result->attnums = (AttrNumber *) ptr;
+ ptr += MAXALIGN(sizeof(AttrNumber) * nkeys);
- /* collations */
- result->collations = (Oid *) ptr;
- ptr += MAXALIGN(sizeof(Oid) * nexprs);
+ /* stats */
+ result->stats = (VacAttrStats **) ptr;
+ ptr += MAXALIGN(sizeof(VacAttrStats *) * nkeys);
/* values */
result->values = (Datum **) ptr;
- ptr += MAXALIGN(sizeof(Datum *) * nexprs);
+ ptr += MAXALIGN(sizeof(Datum *) * nkeys);
/* nulls */
result->nulls = (bool **) ptr;
- ptr += MAXALIGN(sizeof(bool *) * nexprs);
+ ptr += MAXALIGN(sizeof(bool *) * nkeys);
- for (i = 0; i < nexprs; i++)
+ for (i = 0; i < nkeys; i++)
{
result->values[i] = (Datum *) ptr;
ptr += MAXALIGN(sizeof(Datum) * numrows);
@@ -2497,17 +2508,46 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
Assert((ptr - (char *) result) == len);
- result->nexprs = list_length(exprs);
+ /* we have it allocated, so let's fill the values */
+ result->nattnums = nkeys;
+ result->numrows = numrows;
+ /* fill the attribute info - first attributes, then expressions */
idx = 0;
- foreach (lc, exprs)
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->attnums[idx] = k;
+ result->stats[idx] = stats[idx];
+
+ idx++;
+ }
+
+ k = -1;
+ foreach (lc, stat->exprs)
{
Node *expr = (Node *) lfirst(lc);
- result->types[idx] = exprType(expr);
- result->collations[idx] = exprCollation(expr);
+ result->attnums[idx] = k;
+ result->stats[idx] = examine_expression(expr);
idx++;
+ k--;
+ }
+
+ /* first extract values for all the regular attributes */
+ for (i = 0; i < numrows; i++)
+ {
+ idx = 0;
+ k = -1;
+ while ((k = bms_next_member(stat->columns, k)) >= 0)
+ {
+ result->values[idx][i] = heap_getattr(rows[i], k,
+ result->stats[idx]->tupDesc,
+ &result->nulls[idx][i]);
+
+ idx++;
+ }
}
/*
@@ -2526,7 +2566,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
econtext->ecxt_scantuple = slot;
/* Set up expression evaluation state */
- exprstates = ExecPrepareExprList(exprs, estate);
+ exprstates = ExecPrepareExprList(stat->exprs, estate);
for (i = 0; i < numrows; i++)
{
@@ -2539,7 +2579,7 @@ evaluate_expressions(Relation rel, List *exprs, int numrows, HeapTuple *rows)
/* Set up for predicate or expression evaluation */
ExecStoreHeapTuple(rows[i], slot, false);
- idx = 0;
+ idx = bms_num_members(stat->columns);
foreach (lc, exprstates)
{
Datum datum;
diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c
index 323d476814..844ba6f71f 100644
--- a/src/backend/statistics/mcv.c
+++ b/src/backend/statistics/mcv.c
@@ -74,8 +74,7 @@
((ndims) * sizeof(DimensionInfo)) + \
((nitems) * ITEM_SIZE(ndims)))
-static MultiSortSupport build_mss(VacAttrStats **stats, int numattrs,
- ExprInfo *exprs);
+static MultiSortSupport build_mss(StatBuildData *data);
static SortItem *build_distinct_groups(int numrows, SortItem *items,
MultiSortSupport mss, int *ndistinct);
@@ -182,16 +181,11 @@ get_mincount_for_mcv_list(int samplerows, double totalrows)
*
*/
MCVList *
-statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
- Bitmapset *attrs, VacAttrStats **stats,
- double totalrows, int stattarget)
+statext_mcv_build(StatBuildData *data, double totalrows, int stattarget)
{
int i,
- k,
- numattrs,
ngroups,
nitems;
- AttrNumber *attnums;
double mincount;
SortItem *items;
SortItem *groups;
@@ -199,38 +193,11 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
MultiSortSupport mss;
/* comparator for all the columns */
- mss = build_mss(stats, bms_num_members(attrs), exprs);
-
- /*
- * treat expressions as special attributes with high attnums
- *
- * XXX We do this after build_mss, because that expects the bitmapset
- * to only contain simple attributes (with a matching VacAttrStats)
- */
-
- /*
- * Transform the bms into an array, to make accessing i-th member easier.
- */
- attnums = (AttrNumber *) palloc(sizeof(AttrNumber) * (bms_num_members(attrs) + exprs->nexprs));
-
- numattrs = 0;
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- attnums[numattrs++] = k;
-
- /* treat expressions as attributes with negative attnums */
- for (i = 0; i < exprs->nexprs; i++)
- attnums[numattrs++] = -(i+1);
-
- Assert(numattrs >= 2);
- Assert(numattrs == (bms_num_members(attrs) + exprs->nexprs));
-
+ mss = build_mss(data);
/* sort the rows */
- items = build_sorted_items(numrows, &nitems, rows, exprs,
- stats[0]->tupDesc, mss, numattrs, attnums);
+ items = build_sorted_items(data, &nitems, mss,
+ data->nattnums, data->attnums);
if (!items)
return NULL;
@@ -265,7 +232,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* using get_mincount_for_mcv_list() and then keep all items that seem to
* be more common than that.
*/
- mincount = get_mincount_for_mcv_list(numrows, totalrows);
+ mincount = get_mincount_for_mcv_list(data->numrows, totalrows);
/*
* Walk the groups until we find the first group with a count below the
@@ -301,7 +268,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
+ sizeof(SortSupportData));
/* compute frequencies for values in each column */
- nfreqs = (int *) palloc0(sizeof(int) * numattrs);
+ nfreqs = (int *) palloc0(sizeof(int) * data->nattnums);
freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
/*
@@ -312,12 +279,12 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
mcvlist->magic = STATS_MCV_MAGIC;
mcvlist->type = STATS_MCV_TYPE_BASIC;
- mcvlist->ndimensions = numattrs;
+ mcvlist->ndimensions = data->nattnums;
mcvlist->nitems = nitems;
/* store info about data type OIDs */
- for (i = 0; i < numattrs; i++)
- mcvlist->types[i] = stats[i]->attrtypid;
+ for (i = 0; i < data->nattnums; i++)
+ mcvlist->types[i] = data->stats[i]->attrtypid;
/* Copy the first chunk of groups into the result. */
for (i = 0; i < nitems; i++)
@@ -325,22 +292,22 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
/* just pointer to the proper place in the list */
MCVItem *item = &mcvlist->items[i];
- item->values = (Datum *) palloc(sizeof(Datum) * numattrs);
- item->isnull = (bool *) palloc(sizeof(bool) * numattrs);
+ item->values = (Datum *) palloc(sizeof(Datum) * data->nattnums);
+ item->isnull = (bool *) palloc(sizeof(bool) * data->nattnums);
/* copy values for the group */
- memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
- memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
+ memcpy(item->values, groups[i].values, sizeof(Datum) * data->nattnums);
+ memcpy(item->isnull, groups[i].isnull, sizeof(bool) * data->nattnums);
/* groups should be sorted by frequency in descending order */
Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
/* group frequency */
- item->frequency = (double) groups[i].count / numrows;
+ item->frequency = (double) groups[i].count / data->numrows;
/* base frequency, if the attributes were independent */
item->base_frequency = 1.0;
- for (j = 0; j < numattrs; j++)
+ for (j = 0; j < data->nattnums; j++)
{
SortItem *freq;
@@ -356,7 +323,7 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
sizeof(SortItem),
multi_sort_compare, tmp);
- item->base_frequency *= ((double) freq->count) / numrows;
+ item->base_frequency *= ((double) freq->count) / data->numrows;
}
}
@@ -375,17 +342,17 @@ statext_mcv_build(int numrows, HeapTuple *rows, ExprInfo *exprs,
* build MultiSortSupport for the attributes passed in attrs
*/
static MultiSortSupport
-build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
+build_mss(StatBuildData *data)
{
int i;
/* Sort by multiple columns (using array of SortSupport) */
- MultiSortSupport mss = multi_sort_init(numattrs + exprs->nexprs);
+ MultiSortSupport mss = multi_sort_init(data->nattnums);
/* prepare the sort functions for all the attributes */
- for (i = 0; i < numattrs; i++)
+ for (i = 0; i < data->nattnums; i++)
{
- VacAttrStats *colstat = stats[i];
+ VacAttrStats *colstat = data->stats[i];
TypeCacheEntry *type;
type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
@@ -396,20 +363,6 @@ build_mss(VacAttrStats **stats, int numattrs, ExprInfo *exprs)
multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
}
- /* prepare the sort functions for all the expressions */
- for (i = 0; i < exprs->nexprs; i++)
- {
- TypeCacheEntry *type;
-
- type = lookup_type_cache(exprs->types[i], TYPECACHE_LT_OPR);
- if (type->lt_opr == InvalidOid) /* shouldn't happen */
- elog(ERROR, "cache lookup failed for ordering operator for type %u",
- exprs->types[i]);
-
- multi_sort_add_dimension(mss, numattrs + i, type->lt_opr,
- exprs->collations[i]);
- }
-
return mss;
}
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 5e796e7123..7ca59d9785 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -36,9 +36,7 @@
#include "utils/syscache.h"
#include "utils/typcache.h"
-static double ndistinct_for_combination(double totalrows, int numrows,
- HeapTuple *rows, ExprInfo *exprs,
- int nattrs, VacAttrStats **stats,
+static double ndistinct_for_combination(double totalrows, StatBuildData *data,
int k, int *combination);
static double estimate_ndistinct(double totalrows, int numrows, int d, int f1);
static int n_choose_k(int n, int k);
@@ -88,17 +86,12 @@ static void generate_combinations(CombinationGenerator *state);
* allow using Bitmapsets.
*/
MVNDistinct *
-statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats)
+statext_ndistinct_build(double totalrows, StatBuildData *data)
{
MVNDistinct *result;
- int i;
int k;
int itemcnt;
- int numattrs = bms_num_members(attrs);
- int numcombs = num_combinations(numattrs + exprs->nexprs);
- Bitmapset *tmp = NULL;
+ int numcombs = num_combinations(data->nattnums);
result = palloc(offsetof(MVNDistinct, items) +
numcombs * sizeof(MVNDistinctItem));
@@ -106,38 +99,14 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
result->type = STATS_NDISTINCT_TYPE_BASIC;
result->nitems = numcombs;
- /*
- * Treat expressions as system attributes with negative attnums,
- * but offset everything by number of expressions.
- */
- for (i = 0; i < exprs->nexprs; i++)
- {
- AttrNumber attnum = -(i + 1);
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* regular attributes */
- k = -1;
- while ((k = bms_next_member(attrs, k)) >= 0)
- {
- AttrNumber attnum = k;
- tmp = bms_add_member(tmp, attnum + exprs->nexprs);
- }
-
- /* use the newly built bitmapset */
- attrs = tmp;
-
- /* make sure there were no clashes */
- Assert(bms_num_members(attrs) == numattrs + exprs->nexprs);
-
itemcnt = 0;
- for (k = 2; k <= bms_num_members(attrs); k++)
+ for (k = 2; k <= data->nattnums; k++)
{
int *combination;
CombinationGenerator *generator;
/* generate combinations of K out of N elements */
- generator = generator_init(bms_num_members(attrs), k);
+ generator = generator_init(data->nattnums, k);
while ((combination = generator_next(generator)))
{
@@ -147,36 +116,16 @@ statext_ndistinct_build(double totalrows, int numrows, HeapTuple *rows,
item->attributes = palloc(sizeof(AttrNumber) * k);
item->nattributes = k;
+ /* translate the indexes to attnums */
for (j = 0; j < k; j++)
{
- AttrNumber attnum = InvalidAttrNumber;
-
- /*
- * The expressions have negative attnums, so even with the
- * offset are before regular attributes. So the first chunk
- * of indexes are for expressions.
- */
- if (combination[j] >= exprs->nexprs)
- attnum
- = stats[combination[j] - exprs->nexprs]->attr->attnum;
- else
- {
- /* make sure the expression index is valid */
- Assert(combination[j] >= 0);
- Assert(combination[j] < exprs->nexprs);
-
- attnum = -(combination[j] + 1);
- }
-
- Assert(attnum != InvalidAttrNumber);
-
- item->attributes[j] = attnum;
+ item->attributes[j] = data->attnums[combination[j]];
+
+ Assert(AttributeNumberIsValid(item->attributes[j]));
}
item->ndistinct =
- ndistinct_for_combination(totalrows, numrows, rows,
- exprs, numattrs,
- stats, k, combination);
+ ndistinct_for_combination(totalrows, data, k, combination);
itemcnt++;
Assert(itemcnt <= result->nitems);
@@ -471,9 +420,8 @@ pg_ndistinct_send(PG_FUNCTION_ARGS)
* combination of multiple columns.
*/
static double
-ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
- ExprInfo *exprs, int nattrs,
- VacAttrStats **stats, int k, int *combination)
+ndistinct_for_combination(double totalrows, StatBuildData *data,
+ int k, int *combination)
{
int i,
j;
@@ -493,11 +441,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
* using the specified column combination as dimensions. We could try to
* sort in place, but it'd probably be more complex and bug-prone.
*/
- items = (SortItem *) palloc(numrows * sizeof(SortItem));
- values = (Datum *) palloc0(sizeof(Datum) * numrows * k);
- isnull = (bool *) palloc0(sizeof(bool) * numrows * k);
+ items = (SortItem *) palloc(data->numrows * sizeof(SortItem));
+ values = (Datum *) palloc0(sizeof(Datum) * data->numrows * k);
+ isnull = (bool *) palloc0(sizeof(bool) * data->numrows * k);
- for (i = 0; i < numrows; i++)
+ for (i = 0; i < data->numrows; i++)
{
items[i].values = &values[i * k];
items[i].isnull = &isnull[i * k];
@@ -514,24 +462,11 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
{
Oid typid;
TypeCacheEntry *type;
- AttrNumber attnum = InvalidAttrNumber;
- TupleDesc tdesc = NULL;
Oid collid = InvalidOid;
+ VacAttrStats *colstat = data->stats[combination[i]];
- /* first nexprs indexes are for expressions, then regular attributes */
- if (combination[i] >= exprs->nexprs)
- {
- VacAttrStats *colstat = stats[combination[i] - exprs->nexprs];
- typid = colstat->attrtypid;
- attnum = colstat->attr->attnum;
- collid = colstat->attrcollid;
- tdesc = colstat->tupDesc;
- }
- else
- {
- typid = exprs->types[combination[i]];
- collid = exprs->collations[combination[i]];
- }
+ typid = colstat->attrtypid;
+ collid = colstat->attrcollid;
type = lookup_type_cache(typid, TYPECACHE_LT_OPR);
if (type->lt_opr == InvalidOid) /* shouldn't happen */
@@ -542,38 +477,15 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
multi_sort_add_dimension(mss, i, type->lt_opr, collid);
/* accumulate all the data for this dimension into the arrays */
- for (j = 0; j < numrows; j++)
+ for (j = 0; j < data->numrows; j++)
{
- /*
- * The first exprs indexes identify expressions, higher indexes
- * are for plain attributes.
- *
- * XXX This seems a bit strange that we don't offset the (i)
- * in any way?
- */
- if (combination[i] >= exprs->nexprs)
- items[j].values[i] =
- heap_getattr(rows[j],
- attnum,
- tdesc,
- &items[j].isnull[i]);
- else
- {
- /* we know the first nexprs expressions are expressions,
- * and the value is directly the expression index */
- int idx = combination[i];
-
- /* make sure the expression index is valid */
- Assert((idx >= 0) && (idx < exprs->nexprs));
-
- items[j].values[i] = exprs->values[idx][j];
- items[j].isnull[i] = exprs->nulls[idx][j];
- }
+ items[j].values[i] = data->values[combination[i]][j];
+ items[j].isnull[i] = data->nulls[combination[i]][j];
}
}
/* We can sort the array now ... */
- qsort_arg((void *) items, numrows, sizeof(SortItem),
+ qsort_arg((void *) items, data->numrows, sizeof(SortItem),
multi_sort_compare, mss);
/* ... and count the number of distinct combinations */
@@ -581,7 +493,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
f1 = 0;
cnt = 1;
d = 1;
- for (i = 1; i < numrows; i++)
+ for (i = 1; i < data->numrows; i++)
{
if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
{
@@ -598,7 +510,7 @@ ndistinct_for_combination(double totalrows, int numrows, HeapTuple *rows,
if (cnt == 1)
f1 += 1;
- return estimate_ndistinct(totalrows, numrows, d, f1);
+ return estimate_ndistinct(totalrows, data->numrows, d, f1);
}
/* The Duj1 estimator (already used in analyze.c). */
diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h
index 1f09799deb..7acf82aa0e 100644
--- a/src/include/statistics/extended_stats_internal.h
+++ b/src/include/statistics/extended_stats_internal.h
@@ -57,35 +57,26 @@ typedef struct SortItem
int count;
} SortItem;
-/*
- * Used to pass pre-computed information about expressions the stats
- * object is defined on.
- */
-typedef struct ExprInfo
-{
- int nexprs; /* number of expressions */
- Oid *collations; /* collation for each expression */
- Oid *types; /* type of each expression */
- Datum **values; /* values for each expression */
- bool **nulls; /* nulls for each expression */
-} ExprInfo;
-
-extern MVNDistinct *statext_ndistinct_build(double totalrows,
- int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+/* a unified representation of the data the statistics is built on */
+typedef struct StatBuildData {
+ int numrows;
+ int nattnums;
+ AttrNumber *attnums;
+ VacAttrStats **stats;
+ Datum **values;
+ bool **nulls;
+} StatBuildData;
+
+
+extern MVNDistinct *statext_ndistinct_build(double totalrows, StatBuildData *data);
extern bytea *statext_ndistinct_serialize(MVNDistinct *ndistinct);
extern MVNDistinct *statext_ndistinct_deserialize(bytea *data);
-extern MVDependencies *statext_dependencies_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats);
+extern MVDependencies *statext_dependencies_build(StatBuildData *data);
extern bytea *statext_dependencies_serialize(MVDependencies *dependencies);
extern MVDependencies *statext_dependencies_deserialize(bytea *data);
-extern MCVList *statext_mcv_build(int numrows, HeapTuple *rows,
- ExprInfo *exprs, Bitmapset *attrs,
- VacAttrStats **stats,
+extern MCVList *statext_mcv_build(StatBuildData *data,
double totalrows, int stattarget);
extern bytea *statext_mcv_serialize(MCVList *mcv, VacAttrStats **stats);
extern MCVList *statext_mcv_deserialize(bytea *data);
@@ -108,8 +99,7 @@ extern void *bsearch_arg(const void *key, const void *base,
extern AttrNumber *build_attnums_array(Bitmapset *attrs, int nexprs, int *numattrs);
-extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows,
- ExprInfo *exprs, TupleDesc tdesc,
+extern SortItem *build_sorted_items(StatBuildData *data, int *nitems,
MultiSortSupport mss,
int numattrs, AttrNumber *attnums);
--
2.26.2
--------------614DDB87AFFED893713AC0E9--
^ permalink raw reply [nested|flat] 36+ messages in thread
* Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-05-01 15:56 Amonson, Paul D <[email protected]>
0 siblings, 2 replies; 36+ messages in thread
From: Amonson, Paul D @ 2024-05-01 15:56 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
Hi,
Comparing the current SSE4.2 implementation of the CRC32C algorithm in Postgres, to an optimized AVX-512 algorithm [0] we observed significant gains. The result was a ~6.6X average multiplier of increased performance measured on 3 different Intel products. Details below. The AVX-512 algorithm in C is a port of the ISA-L library [1] assembler code.
Workload call size distribution details (write heavy):
* Average was approximately around 1,010 bytes per call
* ~80% of the calls were under 256 bytes
* ~20% of the calls were greater than or equal to 256 bytes up to the max buffer size of 8192
The 256 bytes is important because if the buffer is smaller, it makes sense fallback to the existing implementation. This is because the AVX-512 algorithm needs a minimum of 256 bytes to operate.
Using the above workload data distribution,
at 0% calls < 256 bytes, a 841% improvement on average for crc32c functionality was observed.
at 50% calls < 256 bytes, a 758% improvement on average for crc32c functionality was observed.
at 90% calls < 256 bytes, a 44% improvement on average for crc32c functionality was observed.
at 97.6% calls < 256 bytes, the workload's crc32c performance breaks-even.
at 100% calls < 256 bytes, a 14% regression is seen when using AVX-512 implementation.
The results above are averages over 3 machines, and were measured on: Intel Saphire Rapids bare metal, and using EC2 on AWS cloud: Intel Saphire Rapids (m7i.2xlarge) and Intel Ice Lake (m6i.2xlarge).
Summary Data (Saphire Rapids bare metal, AWS m7i-2xl, and AWS m6i-2xl):
+---------------------+-------------------+-------------------+-------------------+--------------------+
| Rates in Bytes/us | Bare Metal | AWS m6i-2xl | AWS m7i-2xl | |
| (Larger is Better) +---------+---------+---------+---------+---------+---------+ Overall Multiplier |
| | SSE 4.2 | AVX-512 | SSE 4.2 | AVX-512 | SSE 4.2 | AVX-512 | |
+---------------------+---------+---------+---------+---------+---------+---------+--------------------+
| Numbers 256-8192 | 12,046 | 83,196 | 7,471 | 39,965 | 11,867 | 84,589 | 6.62 |
+---------------------+---------+---------+---------+---------+---------+---------+--------------------+
| Numbers 64 - 255 | 16,865 | 15,909 | 9,209 | 7,363 | 12,496 | 10,046 | 0.86 |
+---------------------+---------+---------+---------+---------+---------+---------+--------------------+
| Weighted Multiplier [*] | 1.44 |
+-----------------------------+--------------------+
There was no evidence of AVX-512 frequency throttling from perf data, which stayed steady during the test.
Feedback on this proposed improvement is appreciated. Some questions:
1) This AVX-512 ISA-L derived code uses BSD-3 license [2]. Is this compatible with the PostgreSQL License [3]? They both appear to be very permissive licenses, but I am not an expert on licenses.
2) Is there a preferred benchmark I should run to test this change?
If licensing is a non-issue, I can post the initial patch along with my Postgres benchmark function patch for further review.
Thanks,
Paul
[0] https://www.researchgate.net/publication/263424619_Fast_CRC_computation#full-text
[1] https://github.com/intel/isa-l
[2] https://opensource.org/license/bsd-3-clause
[3] https://opensource.org/license/postgresql
[*] Weights used were 90% of requests less than 256 bytes, 10% greater than or equal to 256 bytes.
^ permalink raw reply [nested|flat] 36+ messages in thread
* RE: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-05-17 16:21 Amonson, Paul D <[email protected]>
parent: Amonson, Paul D <[email protected]>
1 sibling, 1 reply; 36+ messages in thread
From: Amonson, Paul D @ 2024-05-17 16:21 UTC (permalink / raw)
To: [email protected] <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
Hi, forgive the top-post but I have not seen any response to this post?
Thanks,
Paul
> -----Original Message-----
> From: Amonson, Paul D
> Sent: Wednesday, May 1, 2024 8:56 AM
> To: [email protected]
> Cc: Nathan Bossart <[email protected]>; Shankaran, Akash
> <[email protected]>
> Subject: Proposal for Updating CRC32C with AVX-512 Algorithm.
>
> Hi,
>
> Comparing the current SSE4.2 implementation of the CRC32C algorithm in
> Postgres, to an optimized AVX-512 algorithm [0] we observed significant
> gains. The result was a ~6.6X average multiplier of increased performance
> measured on 3 different Intel products. Details below. The AVX-512 algorithm
> in C is a port of the ISA-L library [1] assembler code.
>
> Workload call size distribution details (write heavy):
> * Average was approximately around 1,010 bytes per call
> * ~80% of the calls were under 256 bytes
> * ~20% of the calls were greater than or equal to 256 bytes up to the max
> buffer size of 8192
>
> The 256 bytes is important because if the buffer is smaller, it makes sense
> fallback to the existing implementation. This is because the AVX-512 algorithm
> needs a minimum of 256 bytes to operate.
>
> Using the above workload data distribution,
> at 0% calls < 256 bytes, a 841% improvement on average for crc32c
> functionality was observed.
> at 50% calls < 256 bytes, a 758% improvement on average for crc32c
> functionality was observed.
> at 90% calls < 256 bytes, a 44% improvement on average for crc32c
> functionality was observed.
> at 97.6% calls < 256 bytes, the workload's crc32c performance breaks-even.
> at 100% calls < 256 bytes, a 14% regression is seen when using AVX-512
> implementation.
>
> The results above are averages over 3 machines, and were measured on: Intel
> Saphire Rapids bare metal, and using EC2 on AWS cloud: Intel Saphire Rapids
> (m7i.2xlarge) and Intel Ice Lake (m6i.2xlarge).
>
> Summary Data (Saphire Rapids bare metal, AWS m7i-2xl, and AWS m6i-2xl):
> +---------------------+-------------------+-------------------+-------------------+---------
> -----------+
> | Rates in Bytes/us | Bare Metal | AWS m6i-2xl | AWS m7i-2xl |
> |
> | (Larger is Better) +---------+---------+---------+---------+---------+---------+
> Overall Multiplier |
> | | SSE 4.2 | AVX-512 | SSE 4.2 | AVX-512 | SSE 4.2 | AVX-512 |
> |
> +---------------------+---------+---------+---------+---------+---------+---------+-------
> -------------+
> | Numbers 256-8192 | 12,046 | 83,196 | 7,471 | 39,965 | 11,867 |
> 84,589 | 6.62 |
> +---------------------+---------+---------+---------+---------+---------+---------+-------
> -------------+
> | Numbers 64 - 255 | 16,865 | 15,909 | 9,209 | 7,363 | 12,496 |
> 10,046 | 0.86 |
> +---------------------+---------+---------+---------+---------+---------+---------+-------
> -------------+
> | Weighted Multiplier [*] | 1.44 |
> +-----------------------------+--------------------+
> There was no evidence of AVX-512 frequency throttling from perf data, which
> stayed steady during the test.
>
> Feedback on this proposed improvement is appreciated. Some questions:
> 1) This AVX-512 ISA-L derived code uses BSD-3 license [2]. Is this compatible
> with the PostgreSQL License [3]? They both appear to be very permissive
> licenses, but I am not an expert on licenses.
> 2) Is there a preferred benchmark I should run to test this change?
>
> If licensing is a non-issue, I can post the initial patch along with my Postgres
> benchmark function patch for further review.
>
> Thanks,
> Paul
>
> [0]
> https://www.researchgate.net/publication/263424619_Fast_CRC_computati
> on#full-text
> [1] https://github.com/intel/isa-l
> [2] https://opensource.org/license/bsd-3-clause
> [3] https://opensource.org/license/postgresql
>
> [*] Weights used were 90% of requests less than 256 bytes, 10% greater than
> or equal to 256 bytes.
^ permalink raw reply [nested|flat] 36+ messages in thread
* Re: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-05-20 08:03 Daniel Gustafsson <[email protected]>
parent: Amonson, Paul D <[email protected]>
0 siblings, 1 reply; 36+ messages in thread
From: Daniel Gustafsson @ 2024-05-20 08:03 UTC (permalink / raw)
To: Amonson, Paul D <[email protected]>; +Cc: [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
> On 17 May 2024, at 18:21, Amonson, Paul D <[email protected]> wrote:
> Hi, forgive the top-post but I have not seen any response to this post?
The project is currently in feature-freeze in preparation for the next major
release so new development and ideas are not the top priority right now.
Additionally there is a large developer meeting shortly which many are busy
preparing for. Excercise some patience, and I'm sure there will be follow-ups
to this once development of postgres v18 picks up.
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 36+ messages in thread
* RE: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-06-12 16:43 Amonson, Paul D <[email protected]>
parent: Daniel Gustafsson <[email protected]>
0 siblings, 2 replies; 36+ messages in thread
From: Amonson, Paul D @ 2024-06-12 16:43 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; +Cc: [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
> The project is currently in feature-freeze in preparation for the next major
> release so new development and ideas are not the top priority right now.
> Additionally there is a large developer meeting shortly which many are busy
> preparing for. Excercise some patience, and I'm sure there will be follow-ups
> to this once development of postgres v18 picks up.
Thanks, understood.
I had our OSS internal team, who are experts in OSS licensing, review possible conflicts between the PostgreSQL license and the BSD-Clause 3-like license for the CRC32C AVX-512 code, and they found no issues. Therefore, including the new license into the PostgreSQL codebase should be acceptable.
I am attaching the first official patches. The second patch is a simple test function in PostgreSQL SQL, which I used for testing and benchmarking. It will not be merged.
Code Structure Question: While working on this code, I noticed overlaps with runtime CPU checks done in the previous POPCNT merged code. I was considering that these checks should perhaps be formalized and consolidated into a single source/header file pair. If this is desirable, where should I place these files? Should it be in "src/port" where they are used, or in "src/common" where they are available to all (not just the "src/port" tree)?
Thanks,
Paul
Attachments:
[application/octet-stream] 0001-v2-Feat-Add-AVX512-crc32c-algorithm-to-postgres.patch (42.7K, ../../BL1PR11MB530492830A59F48DB90F4E4ADCC02@BL1PR11MB5304.namprd11.prod.outlook.com/2-0001-v2-Feat-Add-AVX512-crc32c-algorithm-to-postgres.patch)
download | inline diff:
From 067c488c4355ba89f2d5460e72ae44d99229119b Mon Sep 17 00:00:00 2001
From: Paul Amonson <[email protected]>
Date: Tue, 21 May 2024 13:23:39 -0700
Subject: [PATCH] [Feat] Add-AVX512 crc32c algorithm to postgres
Signed-off-by: Paul Amonson <[email protected]>
---
config/c-compiler.m4 | 48 +++++++
configure | 223 +++++++++++++++++++++++------
configure.ac | 106 +++++++++-----
meson.build | 41 +++++-
src/include/pg_config.h.in | 3 +
src/include/port/pg_crc32c.h | 24 +++-
src/port/Makefile | 10 ++
src/port/meson.build | 4 +
src/port/pg_crc32c_avx512.c | 222 ++++++++++++++++++++++++++++
src/port/pg_crc32c_avx512_choose.c | 202 ++++++++++++++++++++++++++
10 files changed, 797 insertions(+), 86 deletions(-)
create mode 100644 src/port/pg_crc32c_avx512.c
create mode 100644 src/port/pg_crc32c_avx512_choose.c
diff --git a/config/c-compiler.m4 b/config/c-compiler.m4
index 10f8c7bd0a..1d33932cb5 100644
--- a/config/c-compiler.m4
+++ b/config/c-compiler.m4
@@ -628,6 +628,54 @@ fi
undefine([Ac_cachevar])dnl
])# PGAC_SSE42_CRC32_INTRINSICS
+# PGAC_AVX512_CRC32_INTRINSICS
+# ---------------------------
+# Check if the compiler supports the x86 CRC instructions added in AVX-512,
+# using the intrinsic functions:
+
+# (We don't test the 8-byte variant, _mm_crc32_u64, but it is assumed to
+# be present if the other ones are, on x86-64 platforms)
+#
+# An optional compiler flag can be passed as arguments (e.g. -msse4.2
+# -mavx512vl -mvpclmulqdq). If the intrinsics are supported, sets
+# pgac_avx512_crc32_intrinsics, and CFLAGS_CRC.
+AC_DEFUN([PGAC_AVX512_CRC32_INTRINSICS],
+[define([Ac_cachevar], [AS_TR_SH([pgac_cv_avx512_crc32_intrinsics_$1])])dnl
+AC_CACHE_CHECK([for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=$1], [Ac_cachevar],
+[pgac_save_CFLAGS=$CFLAGS
+CFLAGS="$pgac_save_CFLAGS $1"
+AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <immintrin.h>],
+ [const unsigned long k1k2[[8]] = {
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86,
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86};
+ unsigned char buffer[[512]];
+ unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L);
+ unsigned long val;
+ __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;
+ __m128i a1, a2;
+ unsigned int crc = 0xffffffff;
+ y8 = _mm512_load_si512((__m512i *)aligned);
+ x0 = _mm512_loadu_si512((__m512i *)k1k2);
+ x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00));
+ x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc)));
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96);
+ a1 = _mm512_extracti32x4_epi32(x1, 3);
+ a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0));
+ x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E);
+ val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0));
+ crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1));
+ return crc != 0;])],
+ [Ac_cachevar=yes],
+ [Ac_cachevar=no])
+CFLAGS="$pgac_save_CFLAGS"])
+if test x"$Ac_cachevar" = x"yes"; then
+ CFLAGS_CRC="$1"
+ pgac_avx512_crc32_intrinsics=yes
+fi
+undefine([Ac_cachevar])dnl
+])# PGAC_AVX512_CRC32_INTRINSICS
+
# PGAC_ARMV8_CRC32C_INTRINSICS
# ----------------------------
diff --git a/configure b/configure
index 7b03db56a6..45cd755867 100755
--- a/configure
+++ b/configure
@@ -14898,7 +14898,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -14944,7 +14944,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -14968,7 +14968,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -15013,7 +15013,7 @@ else
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -15037,7 +15037,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
We can't simply define LARGE_OFF_T to be 9223372036854775807,
since some C++ compilers masquerading as C compilers
incorrectly reject 9223372036854775807. */
-#define LARGE_OFF_T (((off_t) 1 << 62) - 1 + ((off_t) 1 << 62))
+#define LARGE_OFF_T ((((off_t) 1 << 31) << 31) - 1 + (((off_t) 1 << 31) << 31))
int off_t_is_large[(LARGE_OFF_T % 2147483629 == 721
&& LARGE_OFF_T % 2147483647 == 1)
? 1 : -1];
@@ -17774,6 +17774,123 @@ fi
fi
+# Check for Intel AVX-512 intrinsics to do CRC calculations.
+#
+# First check if the _mm512_clmulepi64_epi128 and more intrinsics can
+# be used with the default compiler flags. If not, check if adding
+# the -msse4.2, -mavx512vl and -mvpclmulqdqif flag helps. CFLAGS_CRC
+# is set to -msse4.2, -mavx512vl and -mvpclmulqdqif that's required.
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=" >&5
+$as_echo_n "checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=... " >&6; }
+if ${pgac_cv_avx512_crc32_intrinsics_+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ pgac_save_CFLAGS=$CFLAGS
+CFLAGS="$pgac_save_CFLAGS "
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <immintrin.h>
+int
+main ()
+{
+const unsigned long k1k2[8] = {
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86,
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86};
+ unsigned char buffer[512];
+ unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L);
+ unsigned long val;
+ __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;
+ __m128i a1, a2;
+ unsigned int crc = 0xffffffff;
+ y8 = _mm512_load_si512((__m512i *)aligned);
+ x0 = _mm512_loadu_si512((__m512i *)k1k2);
+ x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00));
+ x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc)));
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96);
+ a1 = _mm512_extracti32x4_epi32(x1, 3);
+ a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0));
+ x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E);
+ val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0));
+ crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1));
+ return crc != 0;
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv_avx512_crc32_intrinsics_=yes
+else
+ pgac_cv_avx512_crc32_intrinsics_=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+CFLAGS="$pgac_save_CFLAGS"
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_avx512_crc32_intrinsics_" >&5
+$as_echo "$pgac_cv_avx512_crc32_intrinsics_" >&6; }
+if test x"$pgac_cv_avx512_crc32_intrinsics_" = x"yes"; then
+ CFLAGS_CRC=""
+ pgac_avx512_crc32_intrinsics=yes
+fi
+
+if test x"$pgac_avx512_crc32_intrinsics" != x"yes"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=-msse4.2 -mavx512vl -mvpclmulqdq" >&5
+$as_echo_n "checking for _mm512_clmulepi64_epi128, _mm512_clmulepi64_epi128... with CFLAGS=-msse4.2 -mavx512vl -mvpclmulqdq... " >&6; }
+if ${pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ pgac_save_CFLAGS=$CFLAGS
+CFLAGS="$pgac_save_CFLAGS -msse4.2 -mavx512vl -mvpclmulqdq"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+#include <immintrin.h>
+int
+main ()
+{
+const unsigned long k1k2[8] = {
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86,
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86};
+ unsigned char buffer[512];
+ unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L);
+ unsigned long val;
+ __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;
+ __m128i a1, a2;
+ unsigned int crc = 0xffffffff;
+ y8 = _mm512_load_si512((__m512i *)aligned);
+ x0 = _mm512_loadu_si512((__m512i *)k1k2);
+ x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00));
+ x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc)));
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96);
+ a1 = _mm512_extracti32x4_epi32(x1, 3);
+ a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0));
+ x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E);
+ val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0));
+ crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1));
+ return crc != 0;
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+ pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq=yes
+else
+ pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+ conftest$ac_exeext conftest.$ac_ext
+CFLAGS="$pgac_save_CFLAGS"
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq" >&5
+$as_echo "$pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq" >&6; }
+if test x"$pgac_cv_avx512_crc32_intrinsics__msse4_2__mavx512vl__mvpclmulqdq" = x"yes"; then
+ CFLAGS_CRC="-msse4.2 -mavx512vl -mvpclmulqdq"
+ pgac_avx512_crc32_intrinsics=yes
+fi
+
+fi
+
# Are we targeting a processor that supports SSE 4.2? gcc, clang and icc all
# define __SSE4_2__ in that case.
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@@ -17946,31 +18063,42 @@ fi
#
# If we are targeting a LoongArch processor, CRC instructions are
# always available (at least on 64 bit), so no runtime check is needed.
-if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then
- # Use Intel SSE 4.2 if available.
- if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then
- USE_SSE42_CRC32C=1
+if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then
+ # Use Intel AVX 512 if available.
+ if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && test x"$AVX512_TARGETED" = x"1" ; then
+ USE_AVX512_CRC32C=1
else
- # Intel SSE 4.2, with runtime check? The CPUID instruction is needed for
- # the runtime check.
- if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then
- USE_SSE42_CRC32C_WITH_RUNTIME_CHECK=1
+ # Use Intel SSE 4.2 if available.
+ if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then
+ USE_SSE42_CRC32C=1
else
- # Use ARM CRC Extension if available.
- if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then
- USE_ARMV8_CRC32C=1
+ # Intel AVX 512, with runtime check? The CPUID instruction is needed for
+ # the runtime check.
+ if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then
+ USE_AVX512_CRC32C_WITH_RUNTIME_CHECK=1
else
- # ARM CRC Extension, with runtime check?
- if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then
- USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK=1
+ # Intel SSE 4.2, with runtime check? The CPUID instruction is needed for
+ # the runtime check.
+ if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then
+ USE_SSE42_CRC32C_WITH_RUNTIME_CHECK=1
else
- # LoongArch CRCC instructions.
- if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then
- USE_LOONGARCH_CRC32C=1
+ # Use ARM CRC Extension if available.
+ if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then
+ USE_ARMV8_CRC32C=1
else
- # fall back to slicing-by-8 algorithm, which doesn't require any
- # special CPU support.
- USE_SLICING_BY_8_CRC32C=1
+ # ARM CRC Extension, with runtime check?
+ if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then
+ USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK=1
+ else
+ # LoongArch CRCC instructions.
+ if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then
+ USE_LOONGARCH_CRC32C=1
+ else
+ # fall back to slicing-by-8 algorithm, which doesn't require any
+ # special CPU support.
+ USE_SLICING_BY_8_CRC32C=1
+ fi
+ fi
fi
fi
fi
@@ -17989,44 +18117,53 @@ $as_echo "#define USE_SSE42_CRC32C 1" >>confdefs.h
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: SSE 4.2" >&5
$as_echo "SSE 4.2" >&6; }
else
- if test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
+ if test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
+
+$as_echo "#define USE_AVX512_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h
+
+ PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_avx512_choose.o"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: AVX 512 with runtime check" >&5
+$as_echo "AVX 512 with runtime check" >&6; }
+ else
+ if test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
$as_echo "#define USE_SSE42_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h
- PG_CRC32C_OBJS="pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_sse42_choose.o"
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: SSE 4.2 with runtime check" >&5
+ PG_CRC32C_OBJS="pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_sse42_choose.o"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: SSE 4.2 with runtime check" >&5
$as_echo "SSE 4.2 with runtime check" >&6; }
- else
- if test x"$USE_ARMV8_CRC32C" = x"1"; then
+ else
+ if test x"$USE_ARMV8_CRC32C" = x"1"; then
$as_echo "#define USE_ARMV8_CRC32C 1" >>confdefs.h
- PG_CRC32C_OBJS="pg_crc32c_armv8.o"
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions" >&5
+ PG_CRC32C_OBJS="pg_crc32c_armv8.o"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions" >&5
$as_echo "ARMv8 CRC instructions" >&6; }
- else
- if test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
+ else
+ if test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
$as_echo "#define USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK 1" >>confdefs.h
- PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o"
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions with runtime check" >&5
+ PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: ARMv8 CRC instructions with runtime check" >&5
$as_echo "ARMv8 CRC instructions with runtime check" >&6; }
- else
- if test x"$USE_LOONGARCH_CRC32C" = x"1"; then
+ else
+ if test x"$USE_LOONGARCH_CRC32C" = x"1"; then
$as_echo "#define USE_LOONGARCH_CRC32C 1" >>confdefs.h
- PG_CRC32C_OBJS="pg_crc32c_loongarch.o"
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: LoongArch CRCC instructions" >&5
+ PG_CRC32C_OBJS="pg_crc32c_loongarch.o"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: LoongArch CRCC instructions" >&5
$as_echo "LoongArch CRCC instructions" >&6; }
- else
+ else
$as_echo "#define USE_SLICING_BY_8_CRC32C 1" >>confdefs.h
- PG_CRC32C_OBJS="pg_crc32c_sb8.o"
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: slicing-by-8" >&5
+ PG_CRC32C_OBJS="pg_crc32c_sb8.o"
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: slicing-by-8" >&5
$as_echo "slicing-by-8" >&6; }
+ fi
fi
fi
fi
diff --git a/configure.ac b/configure.ac
index 63e7be3847..73ea4d95dd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2124,6 +2124,17 @@ if test x"$pgac_sse42_crc32_intrinsics" != x"yes"; then
PGAC_SSE42_CRC32_INTRINSICS([-msse4.2])
fi
+# Check for Intel AVX-512 intrinsics to do CRC calculations.
+#
+# First check if the _mm512_clmulepi64_epi128 and more intrinsics can
+# be used with the default compiler flags. If not, check if adding
+# the -msse4.2, -mavx512vl and -mvpclmulqdqif flag helps. CFLAGS_CRC
+# is set to -msse4.2, -mavx512vl and -mvpclmulqdqif that's required.
+PGAC_AVX512_CRC32_INTRINSICS([])
+if test x"$pgac_avx512_crc32_intrinsics" != x"yes"; then
+ PGAC_AVX512_CRC32_INTRINSICS([-msse4.2 -mavx512vl -mvpclmulqdq])
+fi
+
# Are we targeting a processor that supports SSE 4.2? gcc, clang and icc all
# define __SSE4_2__ in that case.
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [
@@ -2169,31 +2180,42 @@ AC_SUBST(CFLAGS_CRC)
#
# If we are targeting a LoongArch processor, CRC instructions are
# always available (at least on 64 bit), so no runtime check is needed.
-if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then
- # Use Intel SSE 4.2 if available.
- if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then
- USE_SSE42_CRC32C=1
+if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then
+ # Use Intel AVX 512 if available.
+ if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && test x"$AVX512_TARGETED" = x"1" ; then
+ USE_AVX512_CRC32C=1
else
- # Intel SSE 4.2, with runtime check? The CPUID instruction is needed for
- # the runtime check.
- if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then
- USE_SSE42_CRC32C_WITH_RUNTIME_CHECK=1
+ # Use Intel SSE 4.2 if available.
+ if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then
+ USE_SSE42_CRC32C=1
else
- # Use ARM CRC Extension if available.
- if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then
- USE_ARMV8_CRC32C=1
+ # Intel AVX 512, with runtime check? The CPUID instruction is needed for
+ # the runtime check.
+ if test x"$pgac_avx512_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then
+ USE_AVX512_CRC32C_WITH_RUNTIME_CHECK=1
else
- # ARM CRC Extension, with runtime check?
- if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then
- USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK=1
+ # Intel SSE 4.2, with runtime check? The CPUID instruction is needed for
+ # the runtime check.
+ if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && (test x"$pgac_cv__get_cpuid" = x"yes" || test x"$pgac_cv__cpuid" = x"yes"); then
+ USE_SSE42_CRC32C_WITH_RUNTIME_CHECK=1
else
- # LoongArch CRCC instructions.
- if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then
- USE_LOONGARCH_CRC32C=1
+ # Use ARM CRC Extension if available.
+ if test x"$pgac_armv8_crc32c_intrinsics" = x"yes" && test x"$CFLAGS_CRC" = x""; then
+ USE_ARMV8_CRC32C=1
else
- # fall back to slicing-by-8 algorithm, which doesn't require any
- # special CPU support.
- USE_SLICING_BY_8_CRC32C=1
+ # ARM CRC Extension, with runtime check?
+ if test x"$pgac_armv8_crc32c_intrinsics" = x"yes"; then
+ USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK=1
+ else
+ # LoongArch CRCC instructions.
+ if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then
+ USE_LOONGARCH_CRC32C=1
+ else
+ # fall back to slicing-by-8 algorithm, which doesn't require any
+ # special CPU support.
+ USE_SLICING_BY_8_CRC32C=1
+ fi
+ fi
fi
fi
fi
@@ -2208,29 +2230,35 @@ if test x"$USE_SSE42_CRC32C" = x"1"; then
PG_CRC32C_OBJS="pg_crc32c_sse42.o"
AC_MSG_RESULT(SSE 4.2)
else
- if test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
- AC_DEFINE(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check.])
- PG_CRC32C_OBJS="pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_sse42_choose.o"
- AC_MSG_RESULT(SSE 4.2 with runtime check)
+ if test x"$USE_AVX512_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
+ AC_DEFINE(USE_AVX512_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel AVX 512 CRC instructions with a runtime check.])
+ PG_CRC32C_OBJS="pg_crc32c_avx512.o pg_crc32c_sb8.o pg_crc32c_avx512_choose.o"
+ AC_MSG_RESULT(AVX 512 with runtime check)
else
- if test x"$USE_ARMV8_CRC32C" = x"1"; then
- AC_DEFINE(USE_ARMV8_CRC32C, 1, [Define to 1 to use ARMv8 CRC Extension.])
- PG_CRC32C_OBJS="pg_crc32c_armv8.o"
- AC_MSG_RESULT(ARMv8 CRC instructions)
+ if test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
+ AC_DEFINE(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check.])
+ PG_CRC32C_OBJS="pg_crc32c_sse42.o pg_crc32c_sb8.o pg_crc32c_sse42_choose.o"
+ AC_MSG_RESULT(SSE 4.2 with runtime check)
else
- if test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
- AC_DEFINE(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARMv8 CRC Extension with a runtime check.])
- PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o"
- AC_MSG_RESULT(ARMv8 CRC instructions with runtime check)
+ if test x"$USE_ARMV8_CRC32C" = x"1"; then
+ AC_DEFINE(USE_ARMV8_CRC32C, 1, [Define to 1 to use ARMv8 CRC Extension.])
+ PG_CRC32C_OBJS="pg_crc32c_armv8.o"
+ AC_MSG_RESULT(ARMv8 CRC instructions)
else
- if test x"$USE_LOONGARCH_CRC32C" = x"1"; then
- AC_DEFINE(USE_LOONGARCH_CRC32C, 1, [Define to 1 to use LoongArch CRCC instructions.])
- PG_CRC32C_OBJS="pg_crc32c_loongarch.o"
- AC_MSG_RESULT(LoongArch CRCC instructions)
+ if test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
+ AC_DEFINE(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use ARMv8 CRC Extension with a runtime check.])
+ PG_CRC32C_OBJS="pg_crc32c_armv8.o pg_crc32c_sb8.o pg_crc32c_armv8_choose.o"
+ AC_MSG_RESULT(ARMv8 CRC instructions with runtime check)
else
- AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).])
- PG_CRC32C_OBJS="pg_crc32c_sb8.o"
- AC_MSG_RESULT(slicing-by-8)
+ if test x"$USE_LOONGARCH_CRC32C" = x"1"; then
+ AC_DEFINE(USE_LOONGARCH_CRC32C, 1, [Define to 1 to use LoongArch CRCC instructions.])
+ PG_CRC32C_OBJS="pg_crc32c_loongarch.o"
+ AC_MSG_RESULT(LoongArch CRCC instructions)
+ else
+ AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).])
+ PG_CRC32C_OBJS="pg_crc32c_sb8.o"
+ AC_MSG_RESULT(slicing-by-8)
+ fi
fi
fi
fi
diff --git a/meson.build b/meson.build
index f5ca5cfed4..f7e9eb5ecb 100644
--- a/meson.build
+++ b/meson.build
@@ -2109,6 +2109,34 @@ if host_cpu == 'x86' or host_cpu == 'x86_64'
cdata.set('USE_SSE42_CRC32C_WITH_RUNTIME_CHECK', 1)
have_optimized_crc = true
else
+ avx_prog = '''
+#include <immintrin.h>
+
+int main(void)
+{
+ const unsigned long k1k2[8] = {
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86,
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86};
+ unsigned char buffer[512];
+ unsigned char *aligned = (unsigned char*)(((size_t)buffer + 64L) & 0xffffffffffc0L);
+ unsigned long val;
+ __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;
+ __m128i a1, a2;
+ unsigned int crc = 0xffffffff;
+ y8 = _mm512_load_si512((__m512i *)aligned);
+ x0 = _mm512_loadu_si512((__m512i *)k1k2);
+ x1 = _mm512_loadu_si512((__m512i *)(buffer + 0x00));
+ x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc)));
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96);
+ a1 = _mm512_extracti32x4_epi32(x1, 3);
+ a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0));
+ x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E);
+ val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0));
+ crc = (unsigned int)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1));
+ return crc != 0;
+}
+'''
prog = '''
#include <nmmintrin.h>
@@ -2122,13 +2150,20 @@ int main(void)
return crc == 0;
}
'''
-
- if cc.links(prog, name: '_mm_crc32_u8 and _mm_crc32_u32 without -msse4.2',
+ if cc.links(avx_prog,
+ name: '_mm512_clmulepi64_epi128 ... with -msse4.2 -mavx512vl -mvpclmulqdq',
+ args: test_c_args + ['-msse4.2', '-mavx512vl', '-mvpclmulqdq'])
+ cflags_crc += ['-msse4.2','-mavx512vl','-mvpclmulqdq']
+ cdata.set('USE_AVX512_CRC32C', false)
+ cdata.set('USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 1)
+ have_optimized_crc = true
+ endif
+ if have_optimized_crc == false and cc.links(prog, name: '_mm_crc32_u8 and _mm_crc32_u32 without -msse4.2',
args: test_c_args)
# Use Intel SSE 4.2 unconditionally.
cdata.set('USE_SSE42_CRC32C', 1)
have_optimized_crc = true
- elif cc.links(prog, name: '_mm_crc32_u8 and _mm_crc32_u32 with -msse4.2',
+ elif have_optimized_crc == false and cc.links(prog, name: '_mm_crc32_u8 and _mm_crc32_u32 with -msse4.2',
args: test_c_args + ['-msse4.2'])
# Use Intel SSE 4.2, with runtime check. The CPUID instruction is needed for
# the runtime check.
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index f8d3e3b6b8..6e08f1c6c7 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -738,6 +738,9 @@
/* Define to 1 use Intel SSE 4.2 CRC instructions. */
#undef USE_SSE42_CRC32C
+/* Define to 1 to use Intel AVX 512 CRC instructions with a runtime check. */
+#undef USE_AVX512_CRC32C_WITH_RUNTIME_CHECK
+
/* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */
#undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK
diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h
index 63c8e3a00b..b632ac7d59 100644
--- a/src/include/port/pg_crc32c.h
+++ b/src/include/port/pg_crc32c.h
@@ -49,6 +49,14 @@ typedef uint32 pg_crc32c;
extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len);
+#elif defined (USE_AVX512_CRC32)
+/* Use Intel AVX512 instructions. */
+#define COMP_CRC32C(crc, data, len) \
+ ((crc) = pg_comp_crc32c_avx512((crc), (data), (len)))
+#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF)
+
+extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t len);
+
#elif defined(USE_ARMV8_CRC32C)
/* Use ARMv8 CRC Extension instructions. */
@@ -67,6 +75,21 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le
extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_t len);
+#elif defined(USE_AVX512_CRC32C_WITH_RUNTIME_CHECK)
+
+/*
+ * Use Intel AVX-512 instructions, but perform a runtime check first to check that
+ * they are available.
+ */
+#define COMP_CRC32C(crc, data, len) \
+ ((crc) = pg_comp_crc32c((crc), (data), (len)))
+#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF)
+
+extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len);
+extern pg_crc32c (*pg_comp_crc32c)(pg_crc32c crc, const void *data, size_t len);
+
+extern pg_crc32c pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t len);
+
#elif defined(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK) || defined(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK)
/*
@@ -86,7 +109,6 @@ extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t le
#ifdef USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK
extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len);
#endif
-
#else
/*
* Use slicing-by-8 algorithm.
diff --git a/src/port/Makefile b/src/port/Makefile
index db7c02117b..7ae632c6fc 100644
--- a/src/port/Makefile
+++ b/src/port/Makefile
@@ -88,11 +88,21 @@ pg_crc32c_sse42.o: CFLAGS+=$(CFLAGS_CRC)
pg_crc32c_sse42_shlib.o: CFLAGS+=$(CFLAGS_CRC)
pg_crc32c_sse42_srv.o: CFLAGS+=$(CFLAGS_CRC)
+# all versions of pg_crc32c_avx512.o need CFLAGS_CRC
+pg_crc32c_avx512.o: CFLAGS+=$(CFLAGS_CRC)
+pg_crc32c_avx512_shlib.o: CFLAGS+=$(CFLAGS_CRC)
+pg_crc32c_avx512_srv.o: CFLAGS+=$(CFLAGS_CRC)
+
# all versions of pg_crc32c_armv8.o need CFLAGS_CRC
pg_crc32c_armv8.o: CFLAGS+=$(CFLAGS_CRC)
pg_crc32c_armv8_shlib.o: CFLAGS+=$(CFLAGS_CRC)
pg_crc32c_armv8_srv.o: CFLAGS+=$(CFLAGS_CRC)
+# all versions of pg_crc32c_avx512_choose.o need CFLAGS_XSAVE
+pg_crc32c_avx512_choose.o: CFLAGS+=$(CFLAGS_XSAVE)
+pg_crc32c_avx512_choose_shlib.o: CFLAGS+=$(CFLAGS_XSAVE)
+pg_crc32c_avx512_choose_srv.o: CFLAGS+=$(CFLAGS_XSAVE)
+
# all versions of pg_popcount_avx512_choose.o need CFLAGS_XSAVE
pg_popcount_avx512_choose.o: CFLAGS+=$(CFLAGS_XSAVE)
pg_popcount_avx512_choose_shlib.o: CFLAGS+=$(CFLAGS_XSAVE)
diff --git a/src/port/meson.build b/src/port/meson.build
index fd9ee199d1..d635913e9b 100644
--- a/src/port/meson.build
+++ b/src/port/meson.build
@@ -83,6 +83,10 @@ replace_funcs_pos = [
['pg_crc32c_sse42', 'USE_SSE42_CRC32C'],
['pg_crc32c_sse42', 'USE_SSE42_CRC32C_WITH_RUNTIME_CHECK', 'crc'],
['pg_crc32c_sse42_choose', 'USE_SSE42_CRC32C_WITH_RUNTIME_CHECK'],
+ ['pg_crc32c_avx512', 'USE_AVX512_CRC32C'],
+ ['pg_crc32c_avx512', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 'crc'],
+ ['pg_crc32c_avx512_choose', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK', 'xsave'],
+ ['pg_crc32c_sb8', 'USE_AVX512_CRC32C_WITH_RUNTIME_CHECK'],
['pg_crc32c_sb8', 'USE_SSE42_CRC32C_WITH_RUNTIME_CHECK'],
['pg_popcount_avx512', 'USE_AVX512_POPCNT_WITH_RUNTIME_CHECK', 'popcnt'],
['pg_popcount_avx512_choose', 'USE_AVX512_POPCNT_WITH_RUNTIME_CHECK', 'xsave'],
diff --git a/src/port/pg_crc32c_avx512.c b/src/port/pg_crc32c_avx512.c
new file mode 100644
index 0000000000..085c8d99a8
--- /dev/null
+++ b/src/port/pg_crc32c_avx512.c
@@ -0,0 +1,222 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_crc32c_avx512.c
+ * Compute CRC-32C checksum using Intel AVX-512 instructions.
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ * Portions Copyright (c) 2024, Intel(r) Corporation
+ *
+ * IDENTIFICATION
+ * src/port/pg_crc32c_avx512.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "c.h"
+
+#include <immintrin.h>
+
+#include "port/pg_crc32c.h"
+
+/*
+ * Process eight bytes of data at a time.
+ *
+ * NB: We do unaligned accesses here. The Intel architecture allows that,
+ * and performance testing didn't show any performance gain from aligning
+ * the begin address.
+ */
+pg_attribute_no_sanitize_alignment()
+inline
+static
+pg_crc32c
+crc32c_fallback(pg_crc32c crc, const uint8 *p, size_t length)
+{
+ const unsigned char *pend = p + length;
+
+ /*
+ * Process eight bytes of data at a time.
+ *
+ * NB: We do unaligned accesses here. The Intel architecture allows that,
+ * and performance testing didn't show any performance gain from aligning
+ * the begin address.
+ */
+ while (p + 8 <= pend)
+ {
+ crc = (uint32)_mm_crc32_u64(crc, *((const uint64 *)p));
+ p += 8;
+ }
+
+ /* Process remaining full four bytes if any */
+ if (p + 4 <= pend)
+ {
+ crc = _mm_crc32_u32(crc, *((const unsigned int *)p));
+ p += 4;
+ }
+
+ /* Process any remaining bytes one at a time. */
+ while (p < pend)
+ {
+ crc = _mm_crc32_u8(crc, *p);
+ p++;
+ }
+
+ return crc;
+}
+
+/*******************************************************************
+ * pg_crc32c_avx512(): compute the crc32c of the buffer, where the
+ * buffer length must be at least 256, and a multiple of 64. Based
+ * on:
+ *
+ * "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ
+ * Instruction"
+ * V. Gopal, E. Ozturk, et al., 2009,
+ * https://www.researchgate.net/publication/263424619_Fast_CRC_computation#full-text
+ *
+ * This Function:
+ * Copyright 2017 The Chromium Authors
+ * Copyright (c) 2024, Intel(r) Corporation
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the Chromium source repository LICENSE file.
+ * https://chromium.googlesource.com/chromium/src/+/refs/heads/main/LICENSE
+ */
+pg_attribute_no_sanitize_alignment()
+inline
+pg_crc32c
+pg_comp_crc32c_avx512(pg_crc32c crc, const void *data, size_t length)
+{
+ static const uint64 k1k2[8] = {
+ 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86, 0xdcb17aa4,
+ 0xb9e02b86, 0xdcb17aa4, 0xb9e02b86};
+ static const uint64 k3k4[8] = {
+ 0x740eef02, 0x9e4addf8, 0x740eef02, 0x9e4addf8, 0x740eef02,
+ 0x9e4addf8, 0x740eef02, 0x9e4addf8};
+ static const uint64 k9k10[8] = {
+ 0x6992cea2, 0x0d3b6092, 0x6992cea2, 0x0d3b6092, 0x6992cea2,
+ 0x0d3b6092, 0x6992cea2, 0x0d3b6092};
+ static const uint64 k1k4[8] = {
+ 0x1c291d04, 0xddc0152b, 0x3da6d0cb, 0xba4fc28e, 0xf20c0dfe,
+ 0x493c7d27, 0x00000000, 0x00000000};
+
+ const uint8 *input = (const uint8 *)data;
+ if (length >= 256)
+ {
+ uint64 val;
+ __m512i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8;
+ __m128i a1, a2;
+
+ /*
+ * AVX-512 Optimized crc32c algorithm with mimimum of 256 bytes aligned
+ * to 32 bytes.
+ * >>> BEGIN
+ */
+ /*
+ * There's at least one block of 256.
+ */
+ x1 = _mm512_loadu_si512((__m512i *)(input + 0x00));
+ x2 = _mm512_loadu_si512((__m512i *)(input + 0x40));
+ x3 = _mm512_loadu_si512((__m512i *)(input + 0x80));
+ x4 = _mm512_loadu_si512((__m512i *)(input + 0xC0));
+
+ x1 = _mm512_xor_si512(x1, _mm512_castsi128_si512(_mm_cvtsi32_si128(crc)));
+
+ x0 = _mm512_load_si512((__m512i *)k1k2);
+
+ input += 256;
+ length -= 256;
+
+ /*
+ * Parallel fold blocks of 256, if any.
+ */
+ while (length >= 256)
+ {
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x6 = _mm512_clmulepi64_epi128(x2, x0, 0x00);
+ x7 = _mm512_clmulepi64_epi128(x3, x0, 0x00);
+ x8 = _mm512_clmulepi64_epi128(x4, x0, 0x00);
+
+ x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11);
+ x2 = _mm512_clmulepi64_epi128(x2, x0, 0x11);
+ x3 = _mm512_clmulepi64_epi128(x3, x0, 0x11);
+ x4 = _mm512_clmulepi64_epi128(x4, x0, 0x11);
+
+ y5 = _mm512_loadu_si512((__m512i *)(input + 0x00));
+ y6 = _mm512_loadu_si512((__m512i *)(input + 0x40));
+ y7 = _mm512_loadu_si512((__m512i *)(input + 0x80));
+ y8 = _mm512_loadu_si512((__m512i *)(input + 0xC0));
+
+ x1 = _mm512_ternarylogic_epi64(x1, x5, y5, 0x96);
+ x2 = _mm512_ternarylogic_epi64(x2, x6, y6, 0x96);
+ x3 = _mm512_ternarylogic_epi64(x3, x7, y7, 0x96);
+ x4 = _mm512_ternarylogic_epi64(x4, x8, y8, 0x96);
+
+ input += 256;
+ length -= 256;
+ }
+
+ /*
+ * Fold 256 bytes into 64 bytes.
+ */
+ x0 = _mm512_load_si512((__m512i *)k9k10);
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x6 = _mm512_clmulepi64_epi128(x1, x0, 0x11);
+ x3 = _mm512_ternarylogic_epi64(x3, x5, x6, 0x96);
+
+ x7 = _mm512_clmulepi64_epi128(x2, x0, 0x00);
+ x8 = _mm512_clmulepi64_epi128(x2, x0, 0x11);
+ x4 = _mm512_ternarylogic_epi64(x4, x7, x8, 0x96);
+
+ x0 = _mm512_load_si512((__m512i *)k3k4);
+ y5 = _mm512_clmulepi64_epi128(x3, x0, 0x00);
+ y6 = _mm512_clmulepi64_epi128(x3, x0, 0x11);
+ x1 = _mm512_ternarylogic_epi64(x4, y5, y6, 0x96);
+
+ /*
+ * Single fold blocks of 64, if any.
+ */
+ while (length >= 64)
+ {
+ x2 = _mm512_loadu_si512((__m512i *)input);
+
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11);
+ x1 = _mm512_ternarylogic_epi64(x1, x2, x5, 0x96);
+
+ input += 64;
+ length -= 64;
+ }
+
+ /*
+ * Fold 512-bits to 128-bits.
+ */
+ x0 = _mm512_loadu_si512((__m512i *)k1k4);
+
+ a2 = _mm512_extracti32x4_epi32(x1, 3);
+ x5 = _mm512_clmulepi64_epi128(x1, x0, 0x00);
+ x1 = _mm512_clmulepi64_epi128(x1, x0, 0x11);
+ x1 = _mm512_ternarylogic_epi64(x1, x5, _mm512_castsi128_si512(a2), 0x96);
+
+ x0 = _mm512_shuffle_i64x2(x1, x1, 0x4E);
+ x0 = _mm512_xor_epi64(x1, x0);
+ a1 = _mm512_extracti32x4_epi32(x0, 1);
+ a1 = _mm_xor_epi64(a1, _mm512_castsi512_si128(x0));
+
+ /*
+ * Fold 128-bits to 32-bits.
+ */
+ val = _mm_crc32_u64(0, _mm_extract_epi64(a1, 0));
+ crc = (uint32_t)_mm_crc32_u64(val, _mm_extract_epi64(a1, 1));
+ /*
+ * AVX-512 Optimized crc32c algorithm with mimimum of 256 bytes aligned
+ * to 32 bytes.
+ * <<< END
+ ******************************************************************/
+ }
+
+ /*
+ * Finish any remaining bytes.
+ */
+ return crc32c_fallback(crc, input, length);
+}
diff --git a/src/port/pg_crc32c_avx512_choose.c b/src/port/pg_crc32c_avx512_choose.c
new file mode 100644
index 0000000000..d5ccb69d10
--- /dev/null
+++ b/src/port/pg_crc32c_avx512_choose.c
@@ -0,0 +1,202 @@
+/*-------------------------------------------------------------------------
+ *
+ * pg_crc32c_avx512_choose.c
+ * Choose between Intel AVX-512 and software CRC-32C implementation.
+ *
+ * On first call, checks if the CPU we're running on supports Intel AVX-
+ * 512. If it does, use the special AVX-512 instructions for CRC-32C
+ * computation. Otherwise, fall back to the pure software implementation
+ * (slicing-by-8).
+ *
+ * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ * Portions Copyright (c) 2024, Intel(r) Corp.
+ *
+ *
+ * IDENTIFICATION
+ * src/port/pg_crc32c_avx512_choose.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "c.h"
+
+#if defined(HAVE__GET_CPUID) || defined(HAVE__GET_CPUID_COUNT)
+#include <cpuid.h>
+#endif
+
+#ifdef HAVE_XSAVE_INTRINSICS
+#include <immintrin.h>
+#endif
+
+#if defined(HAVE__CPUID) || defined(HAVE__CPUIDEX)
+#include <intrin.h>
+#endif
+
+#include "port/pg_crc32c.h"
+
+typedef unsigned int exx_t;
+
+/*
+ * Helper function.
+ * Test for a bit being set in a exx_t field.
+ */
+inline
+static
+bool
+is_bit_set(exx_t reg, int bit)
+{
+ return (reg & (1 << bit)) != 0;
+}
+
+/*
+ * Intel Platform CPUID check for Linux and Visual Studio platforms.
+ */
+inline
+static
+void
+pg_getcpuid(unsigned int leaf, exx_t *exx)
+{
+#if defined(HAVE__GET_CPUID)
+ __get_cpuid(leaf, &exx[0], &exx[1], &exx[2], &exx[3]);
+#elif defined(HAVE__CPUID)
+ __cpuid(exx, 1);
+#else
+#error cpuid instruction not available
+#endif
+}
+
+/*
+ * Intel Platform CPUIDEX check for Linux and Visual Studio platforms.
+ */
+inline
+static
+void
+pg_getcpuidex(unsigned int leaf, unsigned int subleaf, exx_t *exx)
+{
+#if defined(HAVE__GET_CPUID_COUNT)
+ __get_cpuid_count(leaf, subleaf, &exx[0], &exx[1], &exx[2], &exx[3]);
+#elif defined(HAVE__CPUIDEX)
+ __cpuidex(exx, 7, 0);
+#else
+#error cpuid instruction not available
+#endif
+}
+
+/*
+ * Check for CPU supprt for CPUID: sse4.2
+ */
+inline
+static
+bool
+sse42_available(void)
+{
+ exx_t exx[4] = {0, 0, 0, 0};
+
+ pg_getcpuid(1, exx);
+ return is_bit_set(exx[2], 20); /* sse4.2 */
+}
+
+/*
+ * Check for CPU supprt for CPUID: osxsave
+ */
+inline
+static
+bool
+osxsave_available(void)
+{
+ exx_t exx[4] = {0, 0, 0, 0};
+
+ pg_getcpuid(1, exx);
+ return is_bit_set(exx[2], 27); /* osxsave */
+}
+
+/*
+ * Check for CPU supprt for CPUIDEX: avx512-f
+ */
+inline
+static
+bool
+avx512f_available(void)
+{
+ exx_t exx[4] = {0, 0, 0, 0};
+
+ pg_getcpuidex(7, 0, exx);
+ return is_bit_set(exx[1], 16); /* avx512-f */
+}
+
+/*
+ * Check for CPU supprt for CPUIDEX: vpclmulqdq
+ */
+inline
+static
+bool
+vpclmulqdq_available(void)
+{
+ exx_t exx[4] = {0, 0, 0, 0};
+
+ pg_getcpuidex(7, 0, exx);
+ return is_bit_set(exx[1], 10); /* vpclmulqdq */
+}
+
+/*
+ * Check for CPU supprt for CPUIDEX: vpclmulqdq
+ */
+inline
+static
+bool
+avx512vl_available(void)
+{
+ exx_t exx[4] = {0, 0, 0, 0};
+
+ pg_getcpuidex(7, 0, exx);
+ return is_bit_set(exx[1], 31); /* avx512-vl */
+}
+
+/*
+ * Does XGETBV say the ZMM registers are enabled?
+ *
+ * NB: Caller is responsible for verifying that xsave_available() returns true
+ * before calling this.
+ */
+static inline bool
+zmm_regs_available(void)
+{
+#ifdef HAVE_XSAVE_INTRINSICS
+ return (_xgetbv(0) & 0xe6) == 0xe6;
+#else
+ return false;
+#endif
+}
+
+/*
+ * Returns true if the CPU supports the instructions required for the AVX-512
+ * pg_crc32c implementation.
+ */
+inline
+static
+bool
+pg_crc32c_avx512_available(void)
+{
+ return sse42_available() && osxsave_available() &&
+ avx512f_available() && vpclmulqdq_available() &&
+ avx512vl_available() && zmm_regs_available();
+}
+
+/*
+ * This gets called on the first call. It replaces the function pointer
+ * so that subsequent calls are routed directly to the chosen implementation.
+ */
+static
+pg_crc32c
+pg_comp_avx512_choose(pg_crc32c crc, const void *data, size_t len)
+{
+ if (pg_crc32c_avx512_available())
+ pg_comp_crc32c = pg_comp_crc32c_avx512;
+ else
+ pg_comp_crc32c = pg_comp_crc32c_sb8;
+
+ return pg_comp_crc32c(crc, data, len);
+}
+
+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len) = pg_comp_avx512_choose;
--
2.34.1
[application/octet-stream] 0002-Test-Add-a-Postgres-SQL-function-for-crc32c-testing.patch (3.1K, ../../BL1PR11MB530492830A59F48DB90F4E4ADCC02@BL1PR11MB5304.namprd11.prod.outlook.com/3-0002-Test-Add-a-Postgres-SQL-function-for-crc32c-testing.patch)
download | inline diff:
From 13e19b131002a89b36ea8533f3e05a77c9f6de22 Mon Sep 17 00:00:00 2001
From: Paul Amonson <[email protected]>
Date: Mon, 6 May 2024 08:34:17 -0700
Subject: [PATCH] [Test] Add a Postgres SQL function for crc32c testing.
Signed-off-by: Paul Amonson <[email protected]>
---
src/test/modules/test_crc32c/Makefile | 20 ++++++++++
.../modules/test_crc32c/test_crc32c--1.0.sql | 1 +
src/test/modules/test_crc32c/test_crc32c.c | 39 +++++++++++++++++++
.../modules/test_crc32c/test_crc32c.control | 4 ++
4 files changed, 64 insertions(+)
create mode 100644 src/test/modules/test_crc32c/Makefile
create mode 100644 src/test/modules/test_crc32c/test_crc32c--1.0.sql
create mode 100644 src/test/modules/test_crc32c/test_crc32c.c
create mode 100644 src/test/modules/test_crc32c/test_crc32c.control
diff --git a/src/test/modules/test_crc32c/Makefile b/src/test/modules/test_crc32c/Makefile
new file mode 100644
index 0000000000..5b747c6184
--- /dev/null
+++ b/src/test/modules/test_crc32c/Makefile
@@ -0,0 +1,20 @@
+MODULE_big = test_crc32c
+OBJS = test_crc32c.o
+PGFILEDESC = "test"
+EXTENSION = test_crc32c
+DATA = test_crc32c--1.0.sql
+
+first: all
+
+# test_crc32c.o: CFLAGS+=-g
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_crc32c
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_crc32c/test_crc32c--1.0.sql b/src/test/modules/test_crc32c/test_crc32c--1.0.sql
new file mode 100644
index 0000000000..32f8f0fb2e
--- /dev/null
+++ b/src/test/modules/test_crc32c/test_crc32c--1.0.sql
@@ -0,0 +1 @@
+CREATE FUNCTION drive_crc32c (count int, num int) RETURNS bigint AS 'test_crc32c.so' LANGUAGE C;
diff --git a/src/test/modules/test_crc32c/test_crc32c.c b/src/test/modules/test_crc32c/test_crc32c.c
new file mode 100644
index 0000000000..477f198316
--- /dev/null
+++ b/src/test/modules/test_crc32c/test_crc32c.c
@@ -0,0 +1,41 @@
+/* select drive_crc32c(1000000, 1024); */
+
+#include "postgres.h"
+
+#include "fmgr.h"
+
+#include "port/pg_crc32c.h"
+
+PG_MODULE_MAGIC;
+
+/*
+ * drive_crc32c(count: int, num: int) returns bigint
+ *
+ * count is the nuimber of loops to perform
+ *
+ * num is the number byte in the buffer to calculate
+ * crc32c over.
+ */
+PG_FUNCTION_INFO_V1(drive_crc32c);
+Datum
+drive_crc32c(PG_FUNCTION_ARGS)
+{
+ int64 count = PG_GETARG_INT64(0);
+ int64 num = PG_GETARG_INT64(1);
+ pg_crc32c crc = 0xFFFFFFFF;
+ const char* data = malloc((size_t)num);
+
+ INIT_CRC32C(crc);
+
+ while(count--)
+ {
+ memset((void*)data, count, (size_t)Min(16,num));
+ crc = COMP_CRC32C(crc, data, num);
+ }
+
+ FIN_CRC32C(crc);
+
+ free((void *)data);
+
+ PG_RETURN_INT64((int64_t)crc);
+}
diff --git a/src/test/modules/test_crc32c/test_crc32c.control b/src/test/modules/test_crc32c/test_crc32c.control
new file mode 100644
index 0000000000..878a077ee1
--- /dev/null
+++ b/src/test/modules/test_crc32c/test_crc32c.control
@@ -0,0 +1,4 @@
+comment = 'test'
+default_version = '1.0'
+module_pathname = '$libdir/test_crc32c'
+relocatable = true
--
2.34.1
^ permalink raw reply [nested|flat] 36+ messages in thread
* Re: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-06-12 18:08 Tom Lane <[email protected]>
parent: Amonson, Paul D <[email protected]>
1 sibling, 0 replies; 36+ messages in thread
From: Tom Lane @ 2024-06-12 18:08 UTC (permalink / raw)
To: Amonson, Paul D <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
"Amonson, Paul D" <[email protected]> writes:
> I had our OSS internal team, who are experts in OSS licensing, review possible conflicts between the PostgreSQL license and the BSD-Clause 3-like license for the CRC32C AVX-512 code, and they found no issues. Therefore, including the new license into the PostgreSQL codebase should be acceptable.
Maybe you should get some actual lawyers to answer this type of
question. The Chromium license this code cites is 3-clause-BSD
style, which is NOT compatible: the "advertising" clause is
significant.
In any case, writing copyright notices that are pointers to
external web pages is not how it's done around here. We generally
operate on the assumption that the Postgres source code will
outlive any specific web site. Dead links to incidental material
might be okay, but legally relevant stuff not so much.
regards, tom lane
^ permalink raw reply [nested|flat] 36+ messages in thread
* Re: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-06-12 19:37 Andres Freund <[email protected]>
parent: Amonson, Paul D <[email protected]>
1 sibling, 2 replies; 36+ messages in thread
From: Andres Freund @ 2024-06-12 19:37 UTC (permalink / raw)
To: Amonson, Paul D <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
Hi,
I'm wonder if this isn't going in the wrong direction. We're using CRCs for
something they're not well suited for in my understanding - and are paying a
reasonably high price for it, given that even hardware accelerated CRCs aren't
blazingly fast.
CRCs are used for things like ethernet, iSCSI because they are good at
detecting the kinds of errors encountered, namely short bursts of
bitflips. And the covered data is limited to a fairly small limit.
Which imo makes CRCs a bad choice for WAL. For one, we don't actually expect a
short burst of bitflips, the most likely case is all bits after some point
changing (because only one part of the record made it to disk). For another,
WAL records are *not* limited to a small size, and if anything error detection
becomes more important with longer records (they're likely to be span more
pages / segments).
It's hard to understand, but a nonetheless helpful page is
https://users.ece.cmu.edu/~koopman/crc/crc32.html which lists properties for
crc32c:
https://users.ece.cmu.edu/~koopman/crc/c32/0x8f6e37a0_len.txt
which lists
(0x8f6e37a0; 0x11edc6f41) <=> (0x82f63b78; 0x105ec76f1) {2147483615,2147483615,5243,5243,177,177,47,47,20,20,8,8,6,6,1,1} | gold | (*op) iSCSI; CRC-32C; CRC-32/4
This cryptic notion AFAIU indicates that for our polynomial we can detect 2bit
errors up to a length of 2147483615 bytes, 3 bit errors up to 2147483615, 3
and 4 bit errors up to 5243, 5 and 6 bit errors up to 177, 7/8 bit errors up
to 47.
IMO for our purposes just about all errors are going to be at least at sector
boundaries, i.e. 512 bytes and thus are at least 8 bit large. At that point we
are only guaranteed to find a single-byte error (it'll be common to have
much more) up to a lenght of 47bits. Which isn't a useful guarantee.
With that I perhaps have established that CRC guarantees aren't useful for us.
But not yet why we should use something else: Given that we already aren't
relying on hard guarantees, we could instead just use a fast hash like xxh3.
https://github.com/Cyan4973/xxHash which is fast both for large and small
amounts of data.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 36+ messages in thread
* Re: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-06-12 20:11 Andres Freund <[email protected]>
parent: Amonson, Paul D <[email protected]>
1 sibling, 2 replies; 36+ messages in thread
From: Andres Freund @ 2024-06-12 20:11 UTC (permalink / raw)
To: Amonson, Paul D <[email protected]>; +Cc: [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
Hi,
On 2024-05-01 15:56:08 +0000, Amonson, Paul D wrote:
> Comparing the current SSE4.2 implementation of the CRC32C algorithm in
> Postgres, to an optimized AVX-512 algorithm [0] we observed significant
> gains. The result was a ~6.6X average multiplier of increased performance
> measured on 3 different Intel products. Details below. The AVX-512 algorithm
> in C is a port of the ISA-L library [1] assembler code.
>
> Workload call size distribution details (write heavy):
> * Average was approximately around 1,010 bytes per call
> * ~80% of the calls were under 256 bytes
> * ~20% of the calls were greater than or equal to 256 bytes up to the max buffer size of 8192
This is extremely workload dependent, it's not hard to find workloads with
lots of very small record and very few big ones... What you observed might
have "just" been the warmup behaviour where more full page writes have to be
written.
There a very frequent call computing COMP_CRC32C over just 20 bytes, while
holding a crucial lock. If we were to do introduce something like this
AVX-512 algorithm, it'd probably be worth to dispatch differently in case of
compile-time known small lengths.
How does the latency of the AVX-512 algorithm compare to just using the CRC32C
instruction?
FWIW, I tried the v2 patch on my Xeon Gold 5215 workstation, and dies early on
with SIGILL:
Program terminated with signal SIGILL, Illegal instruction.
#0 0x0000000000d5946c in _mm512_clmulepi64_epi128 (__A=..., __B=..., __C=0)
at /home/andres/build/gcc/master/install/lib/gcc/x86_64-pc-linux-gnu/15/include/vpclmulqdqintrin.h:42
42 return (__m512i) __builtin_ia32_vpclmulqdq_v8di ((__v8di)__A,
(gdb) bt
#0 0x0000000000d5946c in _mm512_clmulepi64_epi128 (__A=..., __B=..., __C=0)
at /home/andres/build/gcc/master/install/lib/gcc/x86_64-pc-linux-gnu/15/include/vpclmulqdqintrin.h:42
#1 pg_comp_crc32c_avx512 (crc=<optimized out>, data=<optimized out>, length=<optimized out>)
at ../../../../../home/andres/src/postgresql/src/port/pg_crc32c_avx512.c:163
#2 0x0000000000819343 in ReadControlFile () at ../../../../../home/andres/src/postgresql/src/backend/access/transam/xlog.c:4375
#3 0x000000000081c4ac in LocalProcessControlFile (reset=<optimized out>) at ../../../../../home/andres/src/postgresql/src/backend/access/transam/xlog.c:4817
#4 0x0000000000a8131d in PostmasterMain (argc=argc@entry=85, argv=argv@entry=0x341b08f0)
at ../../../../../home/andres/src/postgresql/src/backend/postmaster/postmaster.c:902
#5 0x00000000009b53fe in main (argc=85, argv=0x341b08f0) at ../../../../../home/andres/src/postgresql/src/backend/main/main.c:197
Cascade lake doesn't have vpclmulqdq, so we shouldn't be getting here...
This is on an optimied build with meson, with -march=native included in
c_flags.
Relevant configure output:
Checking if "XSAVE intrinsics without -mxsave" : links: NO (cached)
Checking if "XSAVE intrinsics with -mxsave" : links: YES (cached)
Checking if "AVX-512 popcount without -mavx512vpopcntdq -mavx512bw" : links: NO (cached)
Checking if "AVX-512 popcount with -mavx512vpopcntdq -mavx512bw" : links: YES (cached)
Checking if "_mm512_clmulepi64_epi128 ... with -msse4.2 -mavx512vl -mvpclmulqdq" : links: YES
Checking if "x86_64: popcntq instruction" compiles: YES (cached)
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 36+ messages in thread
* RE: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-06-17 22:42 Amonson, Paul D <[email protected]>
parent: Andres Freund <[email protected]>
1 sibling, 0 replies; 36+ messages in thread
From: Amonson, Paul D @ 2024-06-17 22:42 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
> This is extremely workload dependent, it's not hard to find workloads with
> lots of very small record and very few big ones... What you observed might
> have "just" been the warmup behaviour where more full page writes have to
> be written.
Can you tell me how to avoid capturing this "warm-up" so that the numbers are more accurate?
> There a very frequent call computing COMP_CRC32C over just 20 bytes, while
> holding a crucial lock. If we were to do introduce something like this
> AVX-512 algorithm, it'd probably be worth to dispatch differently in case of
> compile-time known small lengths.
So are you suggesting that we be able to directly call into the 64/32 bit based algorithm directly from these known small byte cases in the code? I think that we can do that with a separate API being exposed.
> How does the latency of the AVX-512 algorithm compare to just using the
> CRC32C instruction?
I think I need more information on this one as I am not sure I understand the use case? The same function pointer indirect methods are used with or without the AVX-512 algorithm?
Paul
^ permalink raw reply [nested|flat] 36+ messages in thread
* Re: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-08-08 19:28 Nathan Bossart <[email protected]>
parent: Andres Freund <[email protected]>
1 sibling, 0 replies; 36+ messages in thread
From: Nathan Bossart @ 2024-08-08 19:28 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Amonson, Paul D <[email protected]>; Daniel Gustafsson <[email protected]>; [email protected] <[email protected]>; Shankaran, Akash <[email protected]>
On Wed, Jun 12, 2024 at 12:37:46PM -0700, Andres Freund wrote:
> I'm wonder if this isn't going in the wrong direction. We're using CRCs for
> something they're not well suited for in my understanding - and are paying a
> reasonably high price for it, given that even hardware accelerated CRCs aren't
> blazingly fast.
I tend to agree, especially that we should be more concerned about all
bytes after a certain point being garbage than bit flips. (I think we
should also care about bit flips, but I hope those are much less common
than half-written WAL records.)
> With that I perhaps have established that CRC guarantees aren't useful for us.
> But not yet why we should use something else: Given that we already aren't
> relying on hard guarantees, we could instead just use a fast hash like xxh3.
> https://github.com/Cyan4973/xxHash which is fast both for large and small
> amounts of data.
Would it be out of the question to reuse the page checksum code (i.e., an
FNV-1a derivative)? The chart in your link claims that xxh3 is
substantially faster than "FNV64", but I wonder if the latter was
vectorized. I don't know how our CRC-32C implementations (and proposed
implementations) compare, either.
--
nathan
^ permalink raw reply [nested|flat] 36+ messages in thread
* Re: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-12-04 06:40 John Naylor <[email protected]>
parent: Andres Freund <[email protected]>
1 sibling, 1 reply; 36+ messages in thread
From: John Naylor @ 2024-12-04 06:40 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Amonson, Paul D <[email protected]>; [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
On Thu, Jun 13, 2024 at 3:11 AM Andres Freund <[email protected]> wrote:
>
> On 2024-05-01 15:56:08 +0000, Amonson, Paul D wrote:
> > Workload call size distribution details (write heavy):
> > * Average was approximately around 1,010 bytes per call
> > * ~80% of the calls were under 256 bytes
> > * ~20% of the calls were greater than or equal to 256 bytes up to the max buffer size of 8192
>
> This is extremely workload dependent, it's not hard to find workloads with
> lots of very small record and very few big ones... What you observed might
> have "just" been the warmup behaviour where more full page writes have to be
> written.
Sorry for going back so far, but this thread was pointed out to me,
and this aspect of the design could use some more discussion:
+ * pg_crc32c_avx512(): compute the crc32c of the buffer, where the
+ * buffer length must be at least 256, and a multiple of 64. Based
There is another technique that computes CRC on 3 separate chunks and
combines them at the end, so about 3x faster on large-enough chunks.
That's the way used for the Arm proposal [0], coincidentally also
citing a white paper from Intel, but as Dimitry pointed out in that
thread, its link has apparently disappeared. Raghuveer, do you know
about this, and is there another link available?
http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/crc-iscsi-polynomial-crc32-...
The cut off point in one implementation is only 144 bytes [1] , which
is maybe not as small as we'd like, but is quite a bit smaller than
256. That seems better suited to our workloads, and more portable. I
have a *brand-new* laptop with an Intel chip, and IIUC it doesn't
support AVX-512 because it uses a big-little architecture. I also
understand that Sierra Forrest (a server product line) will be all
little cores with no AVX-512 support, so I'm not sure why the proposal
here requires AVX-512.
> There a very frequent call computing COMP_CRC32C over just 20 bytes, while
> holding a crucial lock. If we were to do introduce something like this
> AVX-512 algorithm, it'd probably be worth to dispatch differently in case of
> compile-time known small lengths.
I know you've read an earlier version of the patch and realized that
it wouldn't help here, but we could probably dispatch differently
regardless, although it may only be worth it if we can inline the
instructions. Since we technically only need to wait for xl_prev, I
believe we could push the computation of the other 12 bytes to before
acquiring the lock, then only execute a single instruction on xl_prev
to complete the CRC computation. Is there any reason why we couldn't
do that, assuming we have a clean way to make that portable? That
would mean that the CRCs between major versions would be different,
but I think we don't guarantee that anyway.
[0] https://commitfest.postgresql.org/50/4620/
[1] https://github.com/komrad36/CRC/blob/master/CRC/golden_intel.cpp#L138C27-L138C42
--
John Naylor
Amazon Web Services
^ permalink raw reply [nested|flat] 36+ messages in thread
* RE: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-12-07 15:16 Devulapalli, Raghuveer <[email protected]>
parent: John Naylor <[email protected]>
0 siblings, 0 replies; 36+ messages in thread
From: Devulapalli, Raghuveer @ 2024-12-07 15:16 UTC (permalink / raw)
To: John Naylor <[email protected]>; Andres Freund <[email protected]>; +Cc: Amonson, Paul D <[email protected]>; [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>
> Sorry for going back so far, but this thread was pointed out to me, and this aspect
> of the design could use some more discussion:
>
> + * pg_crc32c_avx512(): compute the crc32c of the buffer, where the
> + * buffer length must be at least 256, and a multiple of 64. Based
>
> There is another technique that computes CRC on 3 separate chunks and
> combines them at the end, so about 3x faster on large-enough chunks.
> That's the way used for the Arm proposal [0], coincidentally also citing a white
> paper from Intel, but as Dimitry pointed out in that thread, its link has apparently
> disappeared. Raghuveer, do you know about this, and is there another link
> available?
>
> http://www.intel.com/content/dam/www/public/us/en/documents/white-
> papers/crc-iscsi-polynomial-crc32-instruction-paper.pdf
I am not aware of this paper. Let me poke a few people internally and get back to you on this.
> The cut off point in one implementation is only 144 bytes [1] , which is maybe not
> as small as we'd like, but is quite a bit smaller than 256. That seems better suited
> to our workloads, and more portable. I have a *brand-new* laptop with an Intel
> chip, and IIUC it doesn't support AVX-512 because it uses a big-little architecture.
> I also understand that Sierra Forrest (a server product line) will be all little cores
> with no AVX-512 support, so I'm not sure why the proposal here requires AVX-
> 512.
AVX-512 is present all of Intel main P-core based Xeon and AMD's Zen4 and Zen5. Sierra Forest contains the SSE and AVX/AVX2 family ISA but AFAIK AVX/AVX2 does not contain any CRC32C specific instructions. See:
1) https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=pclmul&ig_expand=7...
2) https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#ig_expand=754&techs=AVX...
>
> > There a very frequent call computing COMP_CRC32C over just 20 bytes,
> > while holding a crucial lock. If we were to do introduce something
> > like this
> > AVX-512 algorithm, it'd probably be worth to dispatch differently in
> > case of compile-time known small lengths.
>
> I know you've read an earlier version of the patch and realized that it wouldn't
> help here, but we could probably dispatch differently regardless, although it may
> only be worth it if we can inline the instructions. Since we technically only need to
> wait for xl_prev, I believe we could push the computation of the other 12 bytes to
> before acquiring the lock, then only execute a single instruction on xl_prev to
> complete the CRC computation. Is there any reason why we couldn't do that,
> assuming we have a clean way to make that portable? That would mean that the
> CRCs between major versions would be different, but I think we don't guarantee
> that anyway.
Not sure about that. This is not my expertise and I might need a little time to figure this out. Unfortunately, I am on travel with limited internet connection for the next 6 weeks. I will only be able to address this when I get back. Is this a blocker for the patch or is this something we can address as a revision?
Raghuveer
^ permalink raw reply [nested|flat] 36+ messages in thread
* Re: Proposal for Updating CRC32C with AVX-512 Algorithm.
@ 2024-12-14 05:08 John Naylor <[email protected]>
parent: Andres Freund <[email protected]>
1 sibling, 0 replies; 36+ messages in thread
From: John Naylor @ 2024-12-14 05:08 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; [email protected] <[email protected]>; Nathan Bossart <[email protected]>; Shankaran, Akash <[email protected]>; Devulapalli, Raghuveer <[email protected]>
On Thu, Jun 13, 2024 at 2:37 AM Andres Freund <[email protected]> wrote:
>
> It's hard to understand, but a nonetheless helpful page is
> https://users.ece.cmu.edu/~koopman/crc/crc32.html which lists properties for
> crc32c:
> https://users.ece.cmu.edu/~koopman/crc/c32/0x8f6e37a0_len.txt
> which lists
> (0x8f6e37a0; 0x11edc6f41) <=> (0x82f63b78; 0x105ec76f1) {2147483615,2147483615,5243,5243,177,177,47,47,20,20,8,8,6,6,1,1} | gold | (*op) iSCSI; CRC-32C; CRC-32/4
>
> This cryptic notion AFAIU indicates that for our polynomial we can detect 2bit
> errors up to a length of 2147483615 bytes, 3 bit errors up to 2147483615, 3
> and 4 bit errors up to 5243, 5 and 6 bit errors up to 177, 7/8 bit errors up
> to 47.
One aspect of that cryptic notation that you seemed to have missed is
"(*op)" -- explained as:
*p - primitive polynomial. This has optimal length for HD=3, and good
HD=2 performance above that length.
*o - odd bit errors detected. This has a factor of (x+1) and detects
all odd bit errors (implying that even number of bit errors have an
elevated undetected error rate)
*op - odd bit errors detected plus primitive. This is a primitive
polynomial times (x+1). It has optimal length for HD=4, and detects
all odd bit errors.
This means it's not really a 32-bit checksum -- it's a 1-bit checksum
plus a 31-bit checksum. The 1-bit checksum can detect any odd number
of bit-flips. Do we really want to throw that property away?
Sure, for an even number bitflips beyond a small number, we're left
with the luck ordinary collisions, and CRC is not particularly great,
but for two messages of the same length, I'm also not sure it's all
that bad, either
--
John Naylor
Amazon Web Services
^ permalink raw reply [nested|flat] 36+ messages in thread
end of thread, other threads:[~2024-12-14 05:08 UTC | newest]
Thread overview: 36+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2021-03-04 03:53 [PATCH 5/5] WIP unify handling of attributes and expressions Tomas Vondra <[email protected]>
2024-05-01 15:56 Proposal for Updating CRC32C with AVX-512 Algorithm. Amonson, Paul D <[email protected]>
2024-05-17 16:21 ` RE: Proposal for Updating CRC32C with AVX-512 Algorithm. Amonson, Paul D <[email protected]>
2024-05-20 08:03 ` Re: Proposal for Updating CRC32C with AVX-512 Algorithm. Daniel Gustafsson <[email protected]>
2024-06-12 16:43 ` RE: Proposal for Updating CRC32C with AVX-512 Algorithm. Amonson, Paul D <[email protected]>
2024-06-12 18:08 ` Re: Proposal for Updating CRC32C with AVX-512 Algorithm. Tom Lane <[email protected]>
2024-06-12 19:37 ` Re: Proposal for Updating CRC32C with AVX-512 Algorithm. Andres Freund <[email protected]>
2024-08-08 19:28 ` Re: Proposal for Updating CRC32C with AVX-512 Algorithm. Nathan Bossart <[email protected]>
2024-12-14 05:08 ` Re: Proposal for Updating CRC32C with AVX-512 Algorithm. John Naylor <[email protected]>
2024-06-12 20:11 ` Re: Proposal for Updating CRC32C with AVX-512 Algorithm. Andres Freund <[email protected]>
2024-06-17 22:42 ` RE: Proposal for Updating CRC32C with AVX-512 Algorithm. Amonson, Paul D <[email protected]>
2024-12-04 06:40 ` Re: Proposal for Updating CRC32C with AVX-512 Algorithm. John Naylor <[email protected]>
2024-12-07 15:16 ` RE: Proposal for Updating CRC32C with AVX-512 Algorithm. Devulapalli, Raghuveer <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox