agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH 7/9] Remove the special batch mode, use a larger buffer always 8+ messages / 2 participants [nested] [flat]
* [PATCH 7/9] Remove the special batch mode, use a larger buffer always @ 2021-02-02 00:57 Tomas Vondra <tomas.vondra@postgresql.org> 0 siblings, 0 replies; 8+ messages in thread From: Tomas Vondra @ 2021-02-02 00:57 UTC (permalink / raw) Instead of using a batch mode (with a larger input buffer) only for new ranges, which introduces "special cases" in various places, use it as the standard approach. Also, instead of sizing the buffer to cover the whole range, limit it to some reasonable limit (10x the user-specified size). That should give us most of the benefits without consuming a lot of memory. --- src/backend/access/brin/brin_minmax_multi.c | 853 ++++++++++++-------- 1 file changed, 526 insertions(+), 327 deletions(-) diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c index 69a72da337..fd85c18d83 100644 --- a/src/backend/access/brin/brin_minmax_multi.c +++ b/src/backend/access/brin/brin_minmax_multi.c @@ -57,11 +57,11 @@ #include "access/brin.h" #include "access/brin_internal.h" #include "access/brin_tuple.h" -#include "access/hash.h" /* XXX strange that it fails because of BRIN_AM_OID without this */ #include "access/reloptions.h" #include "access/stratnum.h" #include "access/htup_details.h" #include "catalog/pg_type.h" +#include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "utils/array.h" #include "utils/builtins.h" @@ -92,7 +92,15 @@ */ #define PROCNUM_BASE 11 -#define MINMAX_LOAD_FACTOR 0.75 +/* + * Sizing the insert buffer - we use 10x the number of values specified + * in the reloption, but we cap it to 8192 not to get too large. When + * the buffer gets full, we reduce the number of values by half. + */ +#define MINMAX_BUFFER_FACTOR 10 +#define MINMAX_BUFFER_MIN 256 +#define MINMAX_BUFFER_MAX 8192 +#define MINMAX_BUFFER_LOAD_FACTOR 0.5 typedef struct MinmaxMultiOpaque { @@ -155,23 +163,24 @@ typedef struct Ranges Oid typid; Oid colloid; AttrNumber attno; + FmgrInfo *cmp; /* (2*nranges + nvalues) <= maxvalues */ int nranges; /* number of ranges in the array (stored) */ + int nsorted; /* number of sorted values (ranges + points) */ int nvalues; /* number of values in the data array (all) */ int maxvalues; /* maximum number of values (reloption) */ /* - * In batch mode, we simply add the values into a buffer, without any - * expensive steps (sorting, deduplication, ...). The buffer is sized - * to be larger than the target number of values per range, which - * reduces the number of compactions - operating on larger buffers is - * significantly more efficient, in most cases. We keep the actual - * target and compact to the requested number of values at the very - * end, before serializing to on-disk representation. + * We simply add the values into a large buffer, without any expensive + * steps (sorting, deduplication, ...). The buffer is a multiple of + * the target number of values, so the compaction happen less often, + * amortizing the costs. We keep the actual target and compact to + * the requested number of values at the very end, before serializing + * to on-disk representation. */ - bool batch_mode; - int target_maxvalues; /* requested number of values */ + /* requested number of values */ + int target_maxvalues; /* values stored for this range - either raw values, or ranges */ Datum values[FLEXIBLE_ARRAY_MEMBER]; @@ -203,7 +212,7 @@ typedef struct SerializedRanges static SerializedRanges *range_serialize(Ranges *range); -static Ranges *range_deserialize(SerializedRanges *range); +static Ranges *range_deserialize(int maxvalues, SerializedRanges *range); /* Cache for support and strategy procesures. */ @@ -213,6 +222,14 @@ static FmgrInfo *minmax_multi_get_procinfo(BrinDesc *bdesc, uint16 attno, static FmgrInfo *minmax_multi_get_strategy_procinfo(BrinDesc *bdesc, uint16 attno, Oid subtype, uint16 strategynum); +typedef struct compare_context +{ + FmgrInfo *cmpFn; + Oid colloid; +} compare_context; + +static int compare_values(const void *a, const void *b, void *arg); + /* * minmax_multi_init @@ -240,6 +257,57 @@ minmax_multi_init(int maxvalues) return ranges; } +static void +AssertCheckRanges(Ranges *ranges, FmgrInfo *cmpFn, Oid colloid); + + +static void +range_deduplicate_values(Ranges *range) +{ + int i, n; + int start; + compare_context cxt; + + /* + * If there are no unsorted values, we're done (this probably can't + * happen, as we're adding values to unsorted part). + */ + if (range->nsorted == range->nvalues) + return; + + /* sort the values */ + cxt.colloid = range->colloid; + cxt.cmpFn = range->cmp; + + /* how many values to sort? */ + start = 2 * range->nranges; + + qsort_arg(&range->values[start], + range->nvalues, sizeof(Datum), + compare_values, (void *) &cxt); + + n = 1; + for (i = 1; i < range->nvalues; i++) + { + /* same as preceding value, so store it */ + if (compare_values(&range->values[start + i - 1], + &range->values[start + i], + (void *) &cxt) == 0) + continue; + + range->values[start + n] = range->values[start + i]; + + n++; + } + + /* now all the values are sorted */ + range->nvalues = n; + range->nsorted = n; + + AssertCheckRanges(range, range->cmp, range->colloid); +} + + /* * range_serialize * Serialize the in-memory representation into a compact varlena value. @@ -262,14 +330,25 @@ range_serialize(Ranges *range) /* simple sanity checks */ Assert(range->nranges >= 0); + Assert(range->nsorted >= 0); Assert(range->nvalues >= 0); Assert(range->maxvalues > 0); + Assert(range->target_maxvalues > 0); + + /* at this point the range should be compacted to the target size */ + Assert(2*range->nranges + range->nvalues <= range->target_maxvalues); + + Assert(range->target_maxvalues <= range->maxvalues); + + /* range boundaries are always sorted */ + Assert(range->nvalues >= range->nsorted); + + /* sort and deduplicate values, if there's unsorted part */ + range_deduplicate_values(range); /* see how many Datum values we actually have */ nvalues = 2*range->nranges + range->nvalues; - Assert(2*range->nranges + range->nvalues <= range->maxvalues); - typid = range->typid; typbyval = get_typbyval(typid); typlen = get_typlen(typid); @@ -316,7 +395,7 @@ range_serialize(Ranges *range) serialized->typid = typid; serialized->nranges = range->nranges; serialized->nvalues = range->nvalues; - serialized->maxvalues = range->maxvalues; + serialized->maxvalues = range->target_maxvalues; /* * And now copy also the boundary values (like the length calculation @@ -367,7 +446,7 @@ range_serialize(Ranges *range) * in the in-memory value array. */ static Ranges * -range_deserialize(SerializedRanges *serialized) +range_deserialize(int maxvalues, SerializedRanges *serialized) { int i, nvalues; @@ -384,15 +463,18 @@ range_deserialize(SerializedRanges *serialized) nvalues = 2*serialized->nranges + serialized->nvalues; Assert(nvalues <= serialized->maxvalues); + Assert(serialized->maxvalues <= maxvalues); - range = minmax_multi_init(serialized->maxvalues); + range = minmax_multi_init(maxvalues); /* copy the header info */ range->nranges = serialized->nranges; range->nvalues = serialized->nvalues; - range->maxvalues = serialized->maxvalues; + range->nsorted = serialized->nvalues; + range->maxvalues = maxvalues; + range->target_maxvalues = serialized->maxvalues; + range->typid = serialized->typid; - range->batch_mode = false; typbyval = get_typbyval(serialized->typid); typlen = get_typlen(serialized->typid); @@ -439,12 +521,6 @@ range_deserialize(SerializedRanges *serialized) return range; } -typedef struct compare_context -{ - FmgrInfo *cmpFn; - Oid colloid; -} compare_context; - /* * Used to represent ranges expanded during merging and combining (to * reduce number of boundary values to store). @@ -528,6 +604,115 @@ compare_values(const void *a, const void *b, void *arg) return 0; } +void *bsearch_arg(const void *key, const void *base, + size_t nmemb, size_t size, + int (*compar) (const void *, const void *, void *), + void *arg); + +static bool +has_matching_range(BrinDesc *bdesc, Oid colloid, Ranges *ranges, + Datum newval, AttrNumber attno, Oid typid) +{ + Datum compar; + + Datum minvalue = ranges->values[0]; + Datum maxvalue = ranges->values[2*ranges->nranges - 1]; + + FmgrInfo *cmpLessFn; + FmgrInfo *cmpGreaterFn; + + /* binary search on ranges */ + int start, + end; + + if (ranges->nranges == 0) + return false; + + /* + * Otherwise, need to compare the new value with boundaries of all + * the ranges. First check if it's less than the absolute minimum, + * which is the first value in the array. + */ + cmpLessFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid, + BTLessStrategyNumber); + compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue); + + /* smaller than the smallest value in the range list */ + if (DatumGetBool(compar)) + return false; + + /* + * And now compare it to the existing maximum (last value in the + * data array). But only if we haven't already ruled out a possible + * match in the minvalue check. + */ + cmpGreaterFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid, + BTGreaterStrategyNumber); + compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue); + + if (DatumGetBool(compar)) + return false; + + /* + * So we know it's in the general min/max, the question is whether it + * falls in one of the ranges or gaps. We'll use a binary search on + * the ranges. + * + * it's in the general range, but is it actually covered by any + * of the ranges? Repeat the check for each range. + * + * XXX We simply walk the ranges sequentially, but maybe we could + * further leverage the ordering and non-overlap and use bsearch to + * speed this up a bit. + */ + start = 0; /* first range */ + end = ranges->nranges - 1; /* last range */ + while (true) + { + int midpoint = (start + end) / 2; + + /* this means we ran out of ranges in the last step */ + if (start > end) + return false; + + /* copy the min/max values from the ranges */ + minvalue = ranges->values[2 * midpoint]; + maxvalue = ranges->values[2 * midpoint + 1]; + + /* + * Is the value smaller than the minval? If yes, we'll recurse + * to the left side of range array. + */ + compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue); + + /* smaller than the smallest value in this range */ + if (DatumGetBool(compar)) + { + end = (midpoint - 1); + continue; + } + + /* + * Is the value greater than the minval? If yes, we'll recurse + * to the right side of range array. + */ + compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue); + + /* larger than the largest value in this range */ + if (DatumGetBool(compar)) + { + start = (midpoint + 1); + continue; + } + + /* hey, we found a matching range */ + return true; + } + + return false; +} + + /* * range_contains_value * See if the new value is already contained in the range list. @@ -552,8 +737,6 @@ range_contains_value(BrinDesc *bdesc, Oid colloid, Ranges *ranges, Datum newval) { int i; - FmgrInfo *cmpLessFn; - FmgrInfo *cmpGreaterFn; FmgrInfo *cmpEqualFn; Oid typid = attr->atttypid; @@ -562,77 +745,8 @@ range_contains_value(BrinDesc *bdesc, Oid colloid, * range, and only when there's still a chance of getting a match we * inspect the individual ranges. */ - if (ranges->nranges > 0) - { - Datum compar; - bool match = true; - - Datum minvalue = ranges->values[0]; - Datum maxvalue = ranges->values[2*ranges->nranges - 1]; - - /* - * Otherwise, need to compare the new value with boundaries of all - * the ranges. First check if it's less than the absolute minimum, - * which is the first value in the array. - */ - cmpLessFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid, - BTLessStrategyNumber); - compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue); - - /* smaller than the smallest value in the range list */ - if (DatumGetBool(compar)) - match = false; - - /* - * And now compare it to the existing maximum (last value in the - * data array). But only if we haven't already ruled out a possible - * match in the minvalue check. - */ - if (match) - { - cmpGreaterFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid, - BTGreaterStrategyNumber); - compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue); - - if (DatumGetBool(compar)) - match = false; - } - - /* - * So it's in the general range, but is it actually covered by any - * of the ranges? Repeat the check for each range. - * - * XXX We simply walk the ranges sequentially, but maybe we could - * further leverage the ordering and non-overlap and use bsearch to - * speed this up a bit. - */ - for (i = 0; i < ranges->nranges && match; i++) - { - /* copy the min/max values from the ranges */ - minvalue = ranges->values[2*i]; - maxvalue = ranges->values[2*i+1]; - - /* - * Otherwise, need to compare the new value with boundaries of all - * the ranges. First check if it's less than the absolute minimum, - * which is the first value in the array. - */ - compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue); - - /* smaller than the smallest value in this range */ - if (DatumGetBool(compar)) - continue; - - compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue); - - /* larger than the largest value in this range */ - if (DatumGetBool(compar)) - continue; - - /* hey, we found a matching row */ - return true; - } - } + if (has_matching_range(bdesc, colloid, ranges, newval, attno, typid)) + return true; cmpEqualFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid, BTEqualStrategyNumber); @@ -640,92 +754,42 @@ range_contains_value(BrinDesc *bdesc, Oid colloid, /* * We're done with the ranges, now let's inspect the exact values. * - * XXX Again, we do sequentially search the values - consider leveraging - * the ordering of values to improve performance. + * XXX We do sequential search for small number of values, and bsearch + * once we have more than 16 values. + * + * XXX We only inspect the sorted part - that's OK. For building it may + * produce false negatives, but only after we already added some values + * to the unsorted part, so we've modified the value. And when querying + * the index, there should be no unsorted values. */ - for (i = 2*ranges->nranges; i < 2*ranges->nranges + ranges->nvalues; i++) + if (ranges->nsorted >= 16) { - Datum compar; + compare_context cxt; - compar = FunctionCall2Coll(cmpEqualFn, colloid, newval, ranges->values[i]); + cxt.colloid = ranges->colloid; + cxt.cmpFn = ranges->cmp; - /* found an exact match */ - if (DatumGetBool(compar)) + if (bsearch_arg(&newval, &ranges->values[2*ranges->nranges], + ranges->nsorted, sizeof(Datum), + compare_values, (void *) &cxt) != NULL) return true; } - - /* the value is not covered by this BRIN tuple */ - return false; -} - -/* - * insert_value - * Adds a new value into the single-point part, while maintaining ordering. - * - * The function inserts the new value to the right place in the single-point - * part of the range. It assumes there's enough free space, and then does - * essentially an insert-sort. - * - * XXX Assumes the 'values' array has space for (nvalues+1) entries, and that - * only the first nvalues are used. - */ -static void -insert_value(FmgrInfo *cmp, Oid colloid, Datum *values, int nvalues, - Datum newvalue) -{ - int i; - Datum lt; - - /* If there are no values yet, store the new one and we're done. */ - if (!nvalues) + else { - values[0] = newvalue; - return; - } - - /* - * A common case is that the new value is entirely out of the existing - * range, i.e. it's either smaller or larger than all previous values. - * So we check and handle this case first - first we check the larger - * case, because in that case we can just append the value to the end - * of the array and we're done. - */ + for (i = 2*ranges->nranges; i < 2*ranges->nranges + ranges->nsorted; i++) + { + Datum compar; - /* Is it greater than all existing values in the array? */ - lt = FunctionCall2Coll(cmp, colloid, values[nvalues-1], newvalue); - if (DatumGetBool(lt)) - { - /* just copy it in-place and we're done */ - values[nvalues] = newvalue; - return; - } + compar = FunctionCall2Coll(cmpEqualFn, colloid, newval, ranges->values[i]); - /* - * OK, I lied a bit - we won't check the smaller case explicitly, but - * we'll just compare the value to all existing values in the array. - * But we happen to start with the smallest value, so we're actually - * doing the check anyway. - * - * XXX We do walk the values sequentially. Perhaps we could/should be - * smarter and do some sort of bisection, to improve performance? - */ - for (i = 0; i < nvalues; i++) - { - lt = FunctionCall2Coll(cmp, colloid, newvalue, values[i]); - if (DatumGetBool(lt)) - { - /* - * Move values to make space for the new entry, which should go - * to index 'i'. Entries 0 ... (i-1) should stay where they are. - */ - memmove(&values[i+1], &values[i], (nvalues-i) * sizeof(Datum)); - values[i] = newvalue; - return; + /* found an exact match */ + if (DatumGetBool(compar)) + return true; } } - /* We should never really get here. */ - Assert(false); + /* the value is not covered by this BRIN tuple */ + return false; } #ifdef USE_ASSERT_CHECKING @@ -754,11 +818,12 @@ static void AssertCheckRanges(Ranges *ranges, FmgrInfo *cmpFn, Oid colloid) { #ifdef USE_ASSERT_CHECKING - int i, j; + int i; /* some basic sanity checks */ Assert(ranges->nranges >= 0); - Assert(ranges->nvalues >= 0); + Assert(ranges->nsorted >= 0); + Assert(ranges->nvalues >= ranges->nsorted); Assert(ranges->maxvalues >= 2 * ranges->nranges + ranges->nvalues); Assert(ranges->typid != InvalidOid); @@ -770,32 +835,111 @@ AssertCheckRanges(Ranges *ranges, FmgrInfo *cmpFn, Oid colloid) */ AssertArrayOrder(cmpFn, colloid, ranges->values, 2*ranges->nranges); - /* finally check that none of the values are not covered by ranges */ + /* then the single-point ranges (with nvalues boundar values ) */ + AssertArrayOrder(cmpFn, colloid, &ranges->values[2*ranges->nranges], + ranges->nsorted); + + /* + * Check that none of the values are not covered by ranges (both + * sorted and unsorted) + */ for (i = 0; i < ranges->nvalues; i++) { + Datum compar; + int start, + end; + Datum minvalue, + maxvalue; + Datum value = ranges->values[2 * ranges->nranges + i]; - for (j = 0; j < ranges->nranges; j++) + if (ranges->nranges == 0) + break; + + minvalue = ranges->values[0]; + maxvalue = ranges->values[2*ranges->nranges - 1]; + + /* + * Is the value smaller than the minval? If yes, we'll recurse + * to the left side of range array. + */ + compar = FunctionCall2Coll(cmpFn, colloid, value, minvalue); + + /* smaller than the smallest value in the first range */ + if (DatumGetBool(compar)) + continue; + + /* + * Is the value greater than the minval? If yes, we'll recurse + * to the right side of range array. + */ + compar = FunctionCall2Coll(cmpFn, colloid, maxvalue, value); + + /* larger than the largest value in the last range */ + if (DatumGetBool(compar)) + continue; + + start = 0; /* first range */ + end = ranges->nranges - 1; /* last range */ + while (true) { - Datum r; + int midpoint = (start + end) / 2; + + /* this means we ran out of ranges in the last step */ + if (start > end) + break; + + /* copy the min/max values from the ranges */ + minvalue = ranges->values[2 * midpoint]; + maxvalue = ranges->values[2 * midpoint + 1]; - Datum minval = ranges->values[2 * j]; - Datum maxval = ranges->values[2 * j + 1]; + /* + * Is the value smaller than the minval? If yes, we'll recurse + * to the left side of range array. + */ + compar = FunctionCall2Coll(cmpFn, colloid, value, minvalue); - /* if value is smaller than range minimum, that's OK */ - r = FunctionCall2Coll(cmpFn, colloid, value, minval); - if (DatumGetBool(r)) + /* smaller than the smallest value in this range */ + if (DatumGetBool(compar)) + { + end = (midpoint - 1); continue; + } + + /* + * Is the value greater than the minval? If yes, we'll recurse + * to the right side of range array. + */ + compar = FunctionCall2Coll(cmpFn, colloid, maxvalue, value); - /* if value is greater than range maximum, that's OK */ - r = FunctionCall2Coll(cmpFn, colloid, maxval, value); - if (DatumGetBool(r)) + /* larger than the largest value in this range */ + if (DatumGetBool(compar)) + { + start = (midpoint + 1); continue; + } - /* value is between [min,max], which is wrong */ + /* hey, we found a matching range */ Assert(false); } } + + /* and values in the unsorted part must not be in sorted part */ + for (i = ranges->nsorted; i < ranges->nvalues; i++) + { + compare_context cxt; + Datum value = ranges->values[2 * ranges->nranges + i]; + + if (ranges->nsorted == 0) + break; + + cxt.colloid = ranges->colloid; + cxt.cmpFn = ranges->cmp; + + Assert(bsearch_arg(&value, &ranges->values[2*ranges->nranges], + ranges->nsorted, sizeof(Datum), + compare_values, (void *) &cxt) == NULL); + } #endif } @@ -1106,8 +1250,7 @@ build_distances(FmgrInfo *distanceFn, Oid colloid, */ static CombineRange * build_combine_ranges(FmgrInfo *cmp, Oid colloid, Ranges *ranges, - bool addvalue, Datum newvalue, int *nranges, - bool deduplicate) + int *nranges) { int ncranges; CombineRange *cranges; @@ -1115,28 +1258,15 @@ build_combine_ranges(FmgrInfo *cmp, Oid colloid, Ranges *ranges, /* now do the actual merge sort */ ncranges = ranges->nranges + ranges->nvalues; - /* should we add an extra value? */ - if (addvalue) - ncranges += 1; - cranges = (CombineRange *) palloc0(ncranges * sizeof(CombineRange)); - /* put the new value at the beginning */ - if (addvalue) - { - cranges[0].minval = newvalue; - cranges[0].maxval = newvalue; - cranges[0].collapsed = true; - - /* then the regular and collapsed ranges */ - fill_combine_ranges(&cranges[1], ncranges-1, ranges); - } - else - fill_combine_ranges(cranges, ncranges, ranges); + /* fll the combine ranges */ + fill_combine_ranges(cranges, ncranges, ranges); /* and sort the ranges */ - ncranges = sort_combine_ranges(cmp, colloid, cranges, ncranges, - deduplicate); + ncranges = sort_combine_ranges(cmp, colloid, + cranges, ncranges, + true); /* deduplicate */ /* remember how many cranges we built */ *nranges = ncranges; @@ -1321,19 +1451,28 @@ store_combine_ranges(Ranges *ranges, CombineRange *cranges, int ncranges) } } + /* all the values are sorted */ + ranges->nsorted = ranges->nvalues; + Assert(count_values(cranges, ncranges) == 2*ranges->nranges + ranges->nvalues); Assert(2*ranges->nranges + ranges->nvalues <= ranges->maxvalues); } + + /* - * range_add_value - * Add the new value to the multi-minmax range. + * Consider freeing space in the ranges. + * + * Returns true if the value was actually modified. */ static bool -range_add_value(BrinDesc *bdesc, Oid colloid, - AttrNumber attno, Form_pg_attribute attr, - Ranges *ranges, Datum newval) +ensure_free_space_in_buffer(BrinDesc *bdesc, Oid colloid, + AttrNumber attno, Form_pg_attribute attr, + Ranges *range) { + MemoryContext ctx; + MemoryContext oldctx; + FmgrInfo *cmpFn, *distanceFn; @@ -1342,109 +1481,44 @@ range_add_value(BrinDesc *bdesc, Oid colloid, int ncranges; DistanceValue *distances; - MemoryContext ctx; - MemoryContext oldctx; - - /* we'll certainly need the comparator, so just look it up now */ - cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid, - BTLessStrategyNumber); - - /* comprehensive checks of the input ranges */ - AssertCheckRanges(ranges, cmpFn, colloid); - - Assert((ranges->nranges >= 0) && (ranges->nvalues >= 0) && (ranges->maxvalues >= 0)); - /* - * When batch-building, there should be no ranges. So either the - * number of ranges is 0 or we're not in batching mode. + * If there is free space in the buffer, we're done without having + * to modify anything. */ - Assert(!ranges->batch_mode || (ranges->nranges == 0)); - - /* When batch-building, just add it and we're done. */ - if (ranges->batch_mode) - { - /* there has to be free space, if we've sized the struct */ - Assert(ranges->nvalues < ranges->maxvalues); - - /* Make a copy of the value, if needed. */ - ranges->values[ranges->nvalues++] - = datumCopy(newval, attr->attbyval, attr->attlen);; - - return true; - } - - /* - * Bail out if the value already is covered by the range. - * - * We could also add values until we hit values_per_range, and then - * do the deduplication in a batch, hoping for better efficiency. But - * that would mean we actually modify the range every time, which means - * having to serialize the value, which does palloc, walks the values, - * copies them, etc. Not exactly cheap. - * - * So instead we do the check, which should be fairly cheap - assuming - * the comparator function is not very expensive. - * - * This also implies means the values array can't contain duplicities. - */ - if (range_contains_value(bdesc, colloid, attno, attr, ranges, newval)) + if (2*range->nranges + range->nvalues < range->maxvalues) return false; - /* Make a copy of the value, if needed. */ - newval = datumCopy(newval, attr->attbyval, attr->attlen); - - /* - * If there's space in the values array, copy it in and we're done. - * - * We do want to keep the values sorted (to speed up searches), so we - * do a simple insertion sort. We could do something more elaborate, - * e.g. by sorting the values only now and then, but for small counts - * (e.g. when maxvalues is 64) this should be fine. - */ - if (2*ranges->nranges + ranges->nvalues < ranges->maxvalues) - { - Datum *values; - - /* beginning of the 'single value' part (for convenience) */ - values = &ranges->values[2*ranges->nranges]; - - insert_value(cmpFn, colloid, values, ranges->nvalues, newval); - - ranges->nvalues++; - - /* - * Check we haven't broken the ordering of boundary values (checks - * both parts, but that doesn't hurt). - */ - AssertCheckRanges(ranges, cmpFn, colloid); + /* we'll certainly need the comparator, so just look it up now */ + cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid, + BTLessStrategyNumber); - /* Also check the range contains the value we just added. */ - // FIXME Assert(ranges, cmpFn, colloid); + /* Try deduplicating values in the unsorted part */ + range_deduplicate_values(range); - /* yep, we've modified the range */ + /* did we reduce enough free space by just the deduplication? */ + if (2*range->nranges + range->nvalues <= range->maxvalues * MINMAX_BUFFER_LOAD_FACTOR) return true; - } /* - * Damn - the new value is not in the range yet, but we don't have space - * to just insert it. So we need to combine some of the existing ranges, - * to reduce the number of values we need to store (joining two intervals - * reduces the number of boundaries to store by 2). + * we need to combine some of the existing ranges, to reduce the number + * of values we need to store (joining intervals reduces the number of + * boundary values). * - * To do that we first construct an array of CombineRange items - each - * combine range tracks if it's a regular range or collapsed range, where - * "collapsed" means "single point." + * We first construct an array of CombineRange items - each combine range + * tracks if it's a regular range or a collapsed range, where "collapsed" + * means "single point." This makes the processing easier, as it allows + * handling ranges and points the same way. * - * Existing ranges (we have ranges->nranges of them) map to combine ranges - * directly, while single points (ranges->nvalues of them) have to be - * expanded. We neet the combine ranges to be sorted, and we do that by - * performing a merge sort of ranges, values and new value. + * Then we sort the combine ranges - this is necessary, because although + * ranges and points were sorted on their own, the new array is not. We + * do that by performing a merge sort of the two parts. * * The distanceFn calls (which may internally call e.g. numeric_le) may - * allocate quite a bit of memory, and we must not leak it. Otherwise - * we'd have problems e.g. when building indexes. So we create a local - * memory context and make sure we free the memory before leaving this - * function (not after every call). + * allocate quite a bit of memory, and we must not leak it (we might have + * to do this repeatedly, even for a single BRIN page range). Otherwise + * we'd have problems e.g. when building new indexes. So we use a memory + * context and make sure we free the memory at the end (so if we call + * the distance function many times, it might be an issue, but meh). */ ctx = AllocSetContextCreate(CurrentMemoryContext, "minmax-multi context", @@ -1453,9 +1527,7 @@ range_add_value(BrinDesc *bdesc, Oid colloid, oldctx = MemoryContextSwitchTo(ctx); /* OK build the combine ranges */ - cranges = build_combine_ranges(cmpFn, colloid, ranges, - true, newval, &ncranges, - false); + cranges = build_combine_ranges(cmpFn, colloid, range, &ncranges); /* and we'll also need the 'distance' procedure */ distanceFn = minmax_multi_get_procinfo(bdesc, attno, PROCNUM_DISTANCE); @@ -1469,21 +1541,104 @@ range_add_value(BrinDesc *bdesc, Oid colloid, * use too low or high value. */ ncranges = reduce_combine_ranges(cranges, ncranges, distances, - ranges->maxvalues * MINMAX_LOAD_FACTOR, + range->maxvalues * MINMAX_BUFFER_LOAD_FACTOR, cmpFn, colloid); - Assert(count_values(cranges, ncranges) <= ranges->maxvalues * MINMAX_LOAD_FACTOR); + Assert(count_values(cranges, ncranges) <= range->maxvalues * MINMAX_BUFFER_LOAD_FACTOR); /* decompose the combine ranges into regular ranges and single values */ - store_combine_ranges(ranges, cranges, ncranges); + store_combine_ranges(range, cranges, ncranges); MemoryContextSwitchTo(oldctx); MemoryContextDelete(ctx); /* Did we break the ranges somehow? */ + AssertCheckRanges(range, cmpFn, colloid); + + return true; +} + +/* + * range_add_value + * Add the new value to the multi-minmax range. + */ +static bool +range_add_value(BrinDesc *bdesc, Oid colloid, + AttrNumber attno, Form_pg_attribute attr, + Ranges *ranges, Datum newval) +{ + FmgrInfo *cmpFn; + bool modified = false; + + /* we'll certainly need the comparator, so just look it up now */ + cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid, + BTLessStrategyNumber); + + /* comprehensive checks of the input ranges */ AssertCheckRanges(ranges, cmpFn, colloid); + + /* + * Make sure there's enough free space in the buffer. We only trigger + * this when the buffer is full, which means it had to be modified as + * we size it to be larger than what is stored on disk. + * + * XXX This needs to happen before we check if the value is contained + * in the range, because the value might be in the unsorted part, and + * we don't check that in range_contains_value. The deduplication would + * then move it to the sorted part, and we'd add the value too, which + * violates the rule that we never have duplicates with the ranges + * or sorted values. + * + * XXX At the moment this only does the deduplication. + * + * XXX We might also deduplicate and recheck if the value is contained, + * but that seems like an overkill. We'd need to deduplicate anyway, + * so why not do it now. + */ + modified = ensure_free_space_in_buffer(bdesc, colloid, + attno, attr, ranges); + + /* + * Bail out if the value already is covered by the range. + * + * We could also add values until we hit values_per_range, and then + * do the deduplication in a batch, hoping for better efficiency. But + * that would mean we actually modify the range every time, which means + * having to serialize the value, which does palloc, walks the values, + * copies them, etc. Not exactly cheap. + * + * So instead we do the check, which should be fairly cheap - assuming + * the comparator function is not very expensive. + * + * This also implies means the values array can't contain duplicities. + */ + if (range_contains_value(bdesc, colloid, attno, attr, ranges, newval)) + return modified; + + /* Make a copy of the value, if needed. */ + newval = datumCopy(newval, attr->attbyval, attr->attlen); + + /* + * If there's space in the values array, copy it in and we're done. + * + * We do want to keep the values sorted (to speed up searches), so we + * do a simple insertion sort. We could do something more elaborate, + * e.g. by sorting the values only now and then, but for small counts + * (e.g. when maxvalues is 64) this should be fine. + */ + ranges->values[2*ranges->nranges + ranges->nvalues] = newval; + ranges->nvalues++; + + /* + * Check we haven't broken the ordering of boundary values (checks + * both parts, but that doesn't hurt). + */ + AssertCheckRanges(ranges, cmpFn, colloid); + + /* Also check the range contains the value we just added. */ // FIXME Assert(ranges, cmpFn, colloid); + /* yep, we've modified the range */ return true; } @@ -1506,12 +1661,6 @@ compactify_ranges(BrinDesc *bdesc, Ranges *ranges, int max_values) MemoryContext ctx; MemoryContext oldctx; - /* - * This should only be used in batch mode, and there should be no - * ranges, just individual values. - */ - Assert((ranges->batch_mode) && (ranges->nranges == 0)); - /* we'll certainly need the comparator, so just look it up now */ cmpFn = minmax_multi_get_strategy_procinfo(bdesc, ranges->attno, ranges->typid, BTLessStrategyNumber); @@ -1534,8 +1683,7 @@ compactify_ranges(BrinDesc *bdesc, Ranges *ranges, int max_values) /* OK build the combine ranges */ cranges = build_combine_ranges(cmpFn, ranges->colloid, ranges, - false, (Datum) 0, &ncranges, - true); /* deduplicate */ + &ncranges); /* deduplicate */ if (ncranges > 1) { @@ -1548,7 +1696,7 @@ compactify_ranges(BrinDesc *bdesc, Ranges *ranges, int max_values) * don't expect more tuples to be inserted soon. */ ncranges = reduce_combine_ranges(cranges, ncranges, distances, - max_values, cmpFn, ranges->colloid); + max_values, cmpFn, ranges->colloid); Assert(count_values(cranges, ncranges) <= max_values); } @@ -2052,8 +2200,7 @@ brin_minmax_multi_serialize(BrinDesc *bdesc, Datum src, Datum *dst) * In batch mode, we need to compress the accumulated values to the * actually requested number of values/ranges. */ - if (ranges->batch_mode) - compactify_ranges(bdesc, ranges, ranges->target_maxvalues); + compactify_ranges(bdesc, ranges, ranges->target_maxvalues); s = range_serialize(ranges); dst[0] = PointerGetDatum(s); @@ -2114,15 +2261,39 @@ brin_minmax_multi_add_value(PG_FUNCTION_ARGS) { MemoryContext oldctx; + int target_maxvalues; + int maxvalues; BlockNumber pagesPerRange = BrinGetPagesPerRange(bdesc->bd_index); + /* what was specified as a reloption? */ + target_maxvalues = brin_minmax_multi_get_values(bdesc, opts); + + /* + * Determine the insert buffer size - we use 10x the target, capped + * to the maximum number of values in the heap range. This is more + * than enough, considering the actual number of rows per page is + * likely much lower, but meh. + */ + maxvalues = Min(target_maxvalues * MINMAX_BUFFER_FACTOR, + MaxHeapTuplesPerPage * pagesPerRange); + + /* but always at least the original value */ + maxvalues = Max(maxvalues, target_maxvalues); + + /* always cap by MIN/MAX */ + maxvalues = Max(maxvalues, MINMAX_BUFFER_MIN); + maxvalues = Min(maxvalues, MINMAX_BUFFER_MAX); + oldctx = MemoryContextSwitchTo(column->bv_context); - ranges = minmax_multi_init(MaxHeapTuplesPerPage * pagesPerRange); + ranges = minmax_multi_init(maxvalues); ranges->attno = attno; ranges->colloid = colloid; ranges->typid = attr->atttypid; - ranges->batch_mode = true; - ranges->target_maxvalues = brin_minmax_multi_get_values(bdesc, opts); + ranges->target_maxvalues = target_maxvalues; + + /* we'll certainly need the comparator, so just look it up now */ + ranges->cmp = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid, + BTLessStrategyNumber); MemoryContextSwitchTo(oldctx); @@ -2136,10 +2307,38 @@ brin_minmax_multi_add_value(PG_FUNCTION_ARGS) { MemoryContext oldctx; + int maxvalues; + BlockNumber pagesPerRange = BrinGetPagesPerRange(bdesc->bd_index); + oldctx = MemoryContextSwitchTo(column->bv_context); serialized = (SerializedRanges *) PG_DETOAST_DATUM(column->bv_values[0]); - ranges = range_deserialize(serialized); + + /* + * Determine the insert buffer size - we use 10x the target, capped + * to the maximum number of values in the heap range. This is more + * than enough, considering the actual number of rows per page is + * likely much lower, but meh. + */ + maxvalues = Min(serialized->maxvalues * MINMAX_BUFFER_FACTOR, + MaxHeapTuplesPerPage * pagesPerRange); + + /* but always at least the original value */ + maxvalues = Max(maxvalues, serialized->maxvalues); + + /* always cap by MIN/MAX */ + maxvalues = Max(maxvalues, MINMAX_BUFFER_MIN); + maxvalues = Min(maxvalues, MINMAX_BUFFER_MAX); + + ranges = range_deserialize(maxvalues, serialized); + + ranges->attno = attno; + ranges->colloid = colloid; + ranges->typid = attr->atttypid; + + /* we'll certainly need the comparator, so just look it up now */ + ranges->cmp = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid, + BTLessStrategyNumber); column->bv_mem_value = PointerGetDatum(ranges); column->bv_serialize = brin_minmax_multi_serialize; @@ -2184,7 +2383,7 @@ brin_minmax_multi_consistent(PG_FUNCTION_ARGS) attno = column->bv_attno; serialized = (SerializedRanges *) PG_DETOAST_DATUM(column->bv_values[0]); - ranges = range_deserialize(serialized); + ranges = range_deserialize(serialized->maxvalues, serialized); /* inspect the ranges, and for each one evaluate the scan keys */ for (rangeno = 0; rangeno < ranges->nranges; rangeno++) @@ -2371,8 +2570,8 @@ brin_minmax_multi_union(PG_FUNCTION_ARGS) serialized_a = (SerializedRanges *) PG_DETOAST_DATUM(col_a->bv_values[0]); serialized_b = (SerializedRanges *) PG_DETOAST_DATUM(col_b->bv_values[0]); - ranges_a = range_deserialize(serialized_a); - ranges_b = range_deserialize(serialized_b); + ranges_a = range_deserialize(serialized_a->maxvalues, serialized_a); + ranges_b = range_deserialize(serialized_b->maxvalues, serialized_b); /* make sure neither of the ranges is NULL */ Assert(ranges_a && ranges_b); @@ -2408,7 +2607,7 @@ brin_minmax_multi_union(PG_FUNCTION_ARGS) cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid, BTLessStrategyNumber); - /* sort the combine ranges (don't deduplicate) */ + /* sort the combine ranges (no need to deduplicate) */ sort_combine_ranges(cmpFn, colloid, cranges, ncranges, false); /* @@ -2637,7 +2836,7 @@ brin_minmax_multi_summary_out(PG_FUNCTION_ARGS) fmgr_info(outfunc, &fmgrinfo); /* deserialize the range info easy-to-process pieces */ - ranges_deserialized = range_deserialize(ranges); + ranges_deserialized = range_deserialize(ranges->maxvalues, ranges); appendStringInfo(&str, "nranges: %u nvalues: %u maxvalues: %u", ranges_deserialized->nranges, -- 2.26.2 --------------22A4B241170149838D4D1F8F Content-Type: text/x-patch; charset=UTF-8; name="0008-Define-multi-minmax-oclasses-for-types-with-20210215.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename*0="0008-Define-multi-minmax-oclasses-for-types-with-20210215.pa"; filename*1="tch" ^ permalink raw reply [nested|flat] 8+ messages in thread
* [PATCH v16 4/4] Introduce query_id_const_merge_threshold @ 2023-10-16 14:52 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 8+ messages in thread From: Dmitrii Dolgov @ 2023-10-16 14:52 UTC (permalink / raw) Replace query_id_const_merge with a threshold to allow merging only if the number of elements is larger than specified value, which could be configured using pg_stat_statements parameter query_id_const_merge_threshold. --- .../pg_stat_statements/expected/merging.out | 68 ++++++++++++++++++- .../pg_stat_statements/pg_stat_statements.c | 36 +++++----- contrib/pg_stat_statements/sql/merging.sql | 21 +++++- doc/src/sgml/pgstatstatements.sgml | 19 +++--- src/backend/nodes/queryjumblefuncs.c | 23 +++++-- src/backend/postmaster/postmaster.c | 6 +- src/include/nodes/queryjumble.h | 4 +- 7 files changed, 135 insertions(+), 42 deletions(-) diff --git a/contrib/pg_stat_statements/expected/merging.out b/contrib/pg_stat_statements/expected/merging.out index 7400870f3f6..93d59149bf0 100644 --- a/contrib/pg_stat_statements/expected/merging.out +++ b/contrib/pg_stat_statements/expected/merging.out @@ -36,7 +36,7 @@ SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; (4 rows) -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset(); pg_stat_statements_reset -------------------------- @@ -218,4 +218,68 @@ FROM cte; -------- (0 rows) -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; +SELECT pg_stat_statements_reset(); + pg_stat_statements_reset +-------------------------- + +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +---------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4, $5, $6, $7, $8, $9) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 2 + SELECT pg_stat_statements_reset() | 1 +(3 rows) + +-- With gaps on the threshold +SELECT pg_stat_statements_reset(); + pg_stat_statements_reset +-------------------------- + +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +-------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT pg_stat_statements_reset() | 1 +(2 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 1 + SELECT pg_stat_statements_reset() | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 1 +(4 rows) + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index a5702c3d749..e02171c6767 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -266,8 +266,8 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static ProcessUtility_hook_type prev_ProcessUtility = NULL; -/* An assign hook to keep query_id_const_merge in sync */ -static void pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra); +/* An assign hook to keep query_id_const_merge_threshold in sync */ +static void pgss_query_id_const_merge_assign_hook(int newvalue, void *extra); /* Links to shared memory state */ static pgssSharedState *pgss = NULL; @@ -296,8 +296,8 @@ static bool pgss_track_utility = true; /* whether to track utility commands */ static bool pgss_track_planning = false; /* whether to track planning * duration */ static bool pgss_save = true; /* whether to save stats across shutdown */ -static bool pgss_query_id_const_merge = false; /* request constants merging - * when computing query_id */ +static int pgss_query_id_const_merge_threshold = 0; /* request constants merging + * when computing query_id */ #define pgss_enabled(level) \ (!IsParallelWorker() && \ @@ -459,20 +459,22 @@ _PG_init(void) NULL, NULL); - DefineCustomBoolVariable("pg_stat_statements.query_id_const_merge", - "Whether to merge constants in a list when computing query_id.", - NULL, - &pgss_query_id_const_merge, - false, - PGC_SUSET, - 0, - NULL, - pgss_query_id_const_merge_assign_hook, - NULL); + DefineCustomIntVariable("pg_stat_statements.query_id_const_merge_threshold", + "Whether to merge constants in a list when computing query_id.", + NULL, + &pgss_query_id_const_merge_threshold, + 0, + 0, + INT_MAX, + PGC_SUSET, + 0, + NULL, + pgss_query_id_const_merge_assign_hook, + NULL); MarkGUCPrefixReserved("pg_stat_statements"); - SetQueryIdConstMerge(pgss_query_id_const_merge); + SetQueryIdConstMerge(pgss_query_id_const_merge_threshold); /* * Install hooks. @@ -2956,10 +2958,10 @@ comp_location(const void *a, const void *b) } /* - * Notify query jumbling about query_id_const_merge status + * Notify query jumbling about query_id_const_merge_threshold status */ static void -pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra) +pgss_query_id_const_merge_assign_hook(int newvalue, void *extra) { SetQueryIdConstMerge(newvalue); } diff --git a/contrib/pg_stat_statements/sql/merging.sql b/contrib/pg_stat_statements/sql/merging.sql index c515e48d50c..52ee4fcb216 100644 --- a/contrib/pg_stat_statements/sql/merging.sql +++ b/contrib/pg_stat_statements/sql/merging.sql @@ -15,7 +15,7 @@ SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset(); SELECT * FROM test_merge WHERE id IN (1); @@ -68,4 +68,21 @@ WITH cte AS ( SELECT ARRAY['a', 'b', 'c', const::varchar] AS result FROM cte; -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; + +SELECT pg_stat_statements_reset(); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +-- With gaps on the threshold +SELECT pg_stat_statements_reset(); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index a919696abc2..81e6ab2c0c8 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -549,11 +549,11 @@ In some cases, queries with visibly different texts might get merged into a single <structname>pg_stat_statements</structname> entry. Normally this will happen only for semantically equivalent queries, or if - <xref linkend="guc-query-id-const-merge"/> is enabled and the only + <xref linkend="guc-query-id-const-merge-threshold"/> is greater than 0 and the only difference between queries is the length of an array with constants they contain: <screen> -=# SET query_id_const_merge = on; +=# SET query_id_const_merge_threshold = 1; =# SELECT pg_stat_statements_reset(); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); @@ -881,9 +881,9 @@ calls | 1 <varlistentry> <term> - <varname>pg_stat_statements.query_id_const_merge</varname> (<type>bool</type>) + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> (<type>integer</type>) <indexterm> - <primary><varname>pg_stat_statements.query_id_const_merge</varname> configuration parameter</primary> + <primary><varname>pg_stat_statements.query_id_const_merge_threshold</varname> configuration parameter</primary> </indexterm> </term> @@ -895,11 +895,12 @@ calls | 1 query will get multiple different identifiers, one for each occurrence with an array of different lenght. - If this parameter is on, an array of constants will contribute only the - first element, the last element and the number of elements to the query - identifier. It means two occurences of the same query, where the only - difference is number of constants in the array, are going to get the - same query identifier if the arrays are of similar length. + If this parameter is greater than 0, an array with more than + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> + constants will contribute only the first element, the last element + and the number of elements to the query identifier. It means two + occurences of the same query, where the only difference is number of + constants in the array, are going to get the same query identifier. Such queries are represented in form <literal>'(... [10-99 entries])'</literal>. The parameter could be used to reduce amount of repeating data stored diff --git a/src/backend/nodes/queryjumblefuncs.c b/src/backend/nodes/queryjumblefuncs.c index a1d4567ca66..10be62f1331 100644 --- a/src/backend/nodes/queryjumblefuncs.c +++ b/src/backend/nodes/queryjumblefuncs.c @@ -44,8 +44,8 @@ /* GUC parameters */ int compute_query_id = COMPUTE_QUERY_ID_AUTO; -/* Whether to merge constants in a list when computing query_id */ -bool query_id_const_merge = false; +/* Lower threshold for the list length to merge constants when computing query_id */ +int query_id_const_merge_threshold = 1; /* True when compute_query_id is ON, or AUTO and a module requests them */ bool query_id_enabled = false; @@ -159,12 +159,14 @@ EnableQueryId(void) * Controls constants merging for query identifier computation. * * Third-party plugins can use this function to enable/disable merging - * of constants in a list when query identifier is computed. + * of constants in a list when query identifier is computed. The argument + * specifies the lower threshold for an array length, above which merging will + * be applied. */ void -SetQueryIdConstMerge(bool value) +SetQueryIdConstMerge(int threshold) { - query_id_const_merge = value; + query_id_const_merge_threshold = threshold; } /* @@ -240,7 +242,8 @@ RecordConstLocation(JumbleState *jstate, int location, int magnitude) /* * Verify if the provided list contains could be merged down, which means it - * contains only constant expressions. + * contains only constant expressions and the list contains more than + * query_id_const_merge_threshold elements. * * Return value is the order of magnitude (i.e. how many digits it has) for * length of the list (to use for representation purposes later on) if merging @@ -258,12 +261,18 @@ IsMergeableConstList(List *elements, Const **firstConst, Const **lastConst) if (elements == NULL) return 0; - if (!query_id_const_merge) + if (query_id_const_merge_threshold < 1) { /* Merging is disabled, process everything one by one */ return 0; } + if (elements->length < query_id_const_merge_threshold) + { + /* The list is not large enough */ + return 0; + } + firstExpr = linitial(elements); /* diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 3e5c43ede81..3094d54bab8 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -528,7 +528,7 @@ typedef struct bool redirection_done; bool IsBinaryUpgrade; bool query_id_enabled; - bool query_id_const_merge; + int query_id_const_merge_threshold; int max_safe_fds; int MaxBackends; #ifdef WIN32 @@ -6076,7 +6076,7 @@ save_backend_variables(BackendParameters *param, Port *port, param->redirection_done = redirection_done; param->IsBinaryUpgrade = IsBinaryUpgrade; param->query_id_enabled = query_id_enabled; - param->query_id_const_merge = query_id_const_merge; + param->query_id_const_merge_threshold = query_id_const_merge_threshold; param->max_safe_fds = max_safe_fds; param->MaxBackends = MaxBackends; @@ -6308,7 +6308,7 @@ restore_backend_variables(BackendParameters *param, Port *port) redirection_done = param->redirection_done; IsBinaryUpgrade = param->IsBinaryUpgrade; query_id_enabled = param->query_id_enabled; - query_id_const_merge = param->query_id_const_merge; + query_id_const_merge_threshold = param->query_id_const_merge_threshold; max_safe_fds = param->max_safe_fds; MaxBackends = param->MaxBackends; diff --git a/src/include/nodes/queryjumble.h b/src/include/nodes/queryjumble.h index 8ee2e9afbb6..a9f8cfcbed9 100644 --- a/src/include/nodes/queryjumble.h +++ b/src/include/nodes/queryjumble.h @@ -74,10 +74,10 @@ extern PGDLLIMPORT int compute_query_id; extern const char *CleanQuerytext(const char *query, int *location, int *len); extern JumbleState *JumbleQuery(Query *query); extern void EnableQueryId(void); -extern void SetQueryIdConstMerge(bool value); +extern void SetQueryIdConstMerge(int threshold); extern PGDLLIMPORT bool query_id_enabled; -extern PGDLLIMPORT bool query_id_const_merge; +extern PGDLLIMPORT int query_id_const_merge_threshold; /* * Returns whether query identifier computation has been enabled, either -- 2.41.0 --pjkjqtbeiiimipyr-- ^ permalink raw reply [nested|flat] 8+ messages in thread
* [PATCH v17 4/4] Introduce query_id_const_merge_threshold @ 2024-01-13 13:52 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 8+ messages in thread From: Dmitrii Dolgov @ 2024-01-13 13:52 UTC (permalink / raw) Replace query_id_const_merge with a threshold to allow merging only if the number of elements is larger than specified value, which could be configured using pg_stat_statements parameter query_id_const_merge_threshold. --- .../pg_stat_statements/expected/merging.out | 68 ++++++++++++++++++- .../pg_stat_statements/pg_stat_statements.c | 36 +++++----- contrib/pg_stat_statements/sql/merging.sql | 21 +++++- doc/src/sgml/pgstatstatements.sgml | 23 ++++--- src/backend/nodes/queryjumblefuncs.c | 23 +++++-- src/backend/postmaster/postmaster.c | 6 +- src/include/nodes/queryjumble.h | 4 +- 7 files changed, 137 insertions(+), 44 deletions(-) diff --git a/contrib/pg_stat_statements/expected/merging.out b/contrib/pg_stat_statements/expected/merging.out index 7400870f3f6..93d59149bf0 100644 --- a/contrib/pg_stat_statements/expected/merging.out +++ b/contrib/pg_stat_statements/expected/merging.out @@ -36,7 +36,7 @@ SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; (4 rows) -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset(); pg_stat_statements_reset -------------------------- @@ -218,4 +218,68 @@ FROM cte; -------- (0 rows) -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; +SELECT pg_stat_statements_reset(); + pg_stat_statements_reset +-------------------------- + +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +---------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4, $5, $6, $7, $8, $9) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 2 + SELECT pg_stat_statements_reset() | 1 +(3 rows) + +-- With gaps on the threshold +SELECT pg_stat_statements_reset(); + pg_stat_statements_reset +-------------------------- + +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +-------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT pg_stat_statements_reset() | 1 +(2 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 1 + SELECT pg_stat_statements_reset() | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 1 +(4 rows) + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index a5702c3d749..e02171c6767 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -266,8 +266,8 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static ProcessUtility_hook_type prev_ProcessUtility = NULL; -/* An assign hook to keep query_id_const_merge in sync */ -static void pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra); +/* An assign hook to keep query_id_const_merge_threshold in sync */ +static void pgss_query_id_const_merge_assign_hook(int newvalue, void *extra); /* Links to shared memory state */ static pgssSharedState *pgss = NULL; @@ -296,8 +296,8 @@ static bool pgss_track_utility = true; /* whether to track utility commands */ static bool pgss_track_planning = false; /* whether to track planning * duration */ static bool pgss_save = true; /* whether to save stats across shutdown */ -static bool pgss_query_id_const_merge = false; /* request constants merging - * when computing query_id */ +static int pgss_query_id_const_merge_threshold = 0; /* request constants merging + * when computing query_id */ #define pgss_enabled(level) \ (!IsParallelWorker() && \ @@ -459,20 +459,22 @@ _PG_init(void) NULL, NULL); - DefineCustomBoolVariable("pg_stat_statements.query_id_const_merge", - "Whether to merge constants in a list when computing query_id.", - NULL, - &pgss_query_id_const_merge, - false, - PGC_SUSET, - 0, - NULL, - pgss_query_id_const_merge_assign_hook, - NULL); + DefineCustomIntVariable("pg_stat_statements.query_id_const_merge_threshold", + "Whether to merge constants in a list when computing query_id.", + NULL, + &pgss_query_id_const_merge_threshold, + 0, + 0, + INT_MAX, + PGC_SUSET, + 0, + NULL, + pgss_query_id_const_merge_assign_hook, + NULL); MarkGUCPrefixReserved("pg_stat_statements"); - SetQueryIdConstMerge(pgss_query_id_const_merge); + SetQueryIdConstMerge(pgss_query_id_const_merge_threshold); /* * Install hooks. @@ -2956,10 +2958,10 @@ comp_location(const void *a, const void *b) } /* - * Notify query jumbling about query_id_const_merge status + * Notify query jumbling about query_id_const_merge_threshold status */ static void -pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra) +pgss_query_id_const_merge_assign_hook(int newvalue, void *extra) { SetQueryIdConstMerge(newvalue); } diff --git a/contrib/pg_stat_statements/sql/merging.sql b/contrib/pg_stat_statements/sql/merging.sql index c515e48d50c..52ee4fcb216 100644 --- a/contrib/pg_stat_statements/sql/merging.sql +++ b/contrib/pg_stat_statements/sql/merging.sql @@ -15,7 +15,7 @@ SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset(); SELECT * FROM test_merge WHERE id IN (1); @@ -68,4 +68,21 @@ WITH cte AS ( SELECT ARRAY['a', 'b', 'c', const::varchar] AS result FROM cte; -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; + +SELECT pg_stat_statements_reset(); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +-- With gaps on the threshold +SELECT pg_stat_statements_reset(); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index ff24153c493..5545accc560 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -549,12 +549,12 @@ In some cases, queries with visibly different texts might get merged into a single <structname>pg_stat_statements</structname> entry. Normally this will happen only for semantically equivalent queries, or if - <varname>pg_stat_statements.query_id_const_merge</varname> is enabled and - the only difference between queries is the length of an array with constants - they contain: + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> is + enabled and the only difference between queries is the length of an array + with constants they contain: <screen> -=# SET query_id_const_merge = on; +=# SET query_id_const_merge_threshold = 1; =# SELECT pg_stat_statements_reset(); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); @@ -882,9 +882,9 @@ calls | 1 <varlistentry> <term> - <varname>pg_stat_statements.query_id_const_merge</varname> (<type>bool</type>) + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> (<type>integer</type>) <indexterm> - <primary><varname>pg_stat_statements.query_id_const_merge</varname> configuration parameter</primary> + <primary><varname>pg_stat_statements.query_id_const_merge_threshold</varname> configuration parameter</primary> </indexterm> </term> @@ -896,11 +896,12 @@ calls | 1 query will get multiple different identifiers, one for each occurrence with an array of different lenght. - If this parameter is on, an array of constants will contribute only the - first element, the last element and the number of elements to the query - identifier. It means two occurences of the same query, where the only - difference is number of constants in the array, are going to get the - same query identifier if the arrays are of similar length. + If this parameter is greater than 0, an array with more than + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> + constants will contribute only the first element, the last element + and the number of elements to the query identifier. It means two + occurences of the same query, where the only difference is number of + constants in the array, are going to get the same query identifier. Such queries are represented in form <literal>'(... [10-99 entries])'</literal>. The parameter could be used to reduce amount of repeating data stored diff --git a/src/backend/nodes/queryjumblefuncs.c b/src/backend/nodes/queryjumblefuncs.c index a1d4567ca66..10be62f1331 100644 --- a/src/backend/nodes/queryjumblefuncs.c +++ b/src/backend/nodes/queryjumblefuncs.c @@ -44,8 +44,8 @@ /* GUC parameters */ int compute_query_id = COMPUTE_QUERY_ID_AUTO; -/* Whether to merge constants in a list when computing query_id */ -bool query_id_const_merge = false; +/* Lower threshold for the list length to merge constants when computing query_id */ +int query_id_const_merge_threshold = 1; /* True when compute_query_id is ON, or AUTO and a module requests them */ bool query_id_enabled = false; @@ -159,12 +159,14 @@ EnableQueryId(void) * Controls constants merging for query identifier computation. * * Third-party plugins can use this function to enable/disable merging - * of constants in a list when query identifier is computed. + * of constants in a list when query identifier is computed. The argument + * specifies the lower threshold for an array length, above which merging will + * be applied. */ void -SetQueryIdConstMerge(bool value) +SetQueryIdConstMerge(int threshold) { - query_id_const_merge = value; + query_id_const_merge_threshold = threshold; } /* @@ -240,7 +242,8 @@ RecordConstLocation(JumbleState *jstate, int location, int magnitude) /* * Verify if the provided list contains could be merged down, which means it - * contains only constant expressions. + * contains only constant expressions and the list contains more than + * query_id_const_merge_threshold elements. * * Return value is the order of magnitude (i.e. how many digits it has) for * length of the list (to use for representation purposes later on) if merging @@ -258,12 +261,18 @@ IsMergeableConstList(List *elements, Const **firstConst, Const **lastConst) if (elements == NULL) return 0; - if (!query_id_const_merge) + if (query_id_const_merge_threshold < 1) { /* Merging is disabled, process everything one by one */ return 0; } + if (elements->length < query_id_const_merge_threshold) + { + /* The list is not large enough */ + return 0; + } + firstExpr = linitial(elements); /* diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 3e5c43ede81..3094d54bab8 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -528,7 +528,7 @@ typedef struct bool redirection_done; bool IsBinaryUpgrade; bool query_id_enabled; - bool query_id_const_merge; + int query_id_const_merge_threshold; int max_safe_fds; int MaxBackends; #ifdef WIN32 @@ -6076,7 +6076,7 @@ save_backend_variables(BackendParameters *param, Port *port, param->redirection_done = redirection_done; param->IsBinaryUpgrade = IsBinaryUpgrade; param->query_id_enabled = query_id_enabled; - param->query_id_const_merge = query_id_const_merge; + param->query_id_const_merge_threshold = query_id_const_merge_threshold; param->max_safe_fds = max_safe_fds; param->MaxBackends = MaxBackends; @@ -6308,7 +6308,7 @@ restore_backend_variables(BackendParameters *param, Port *port) redirection_done = param->redirection_done; IsBinaryUpgrade = param->IsBinaryUpgrade; query_id_enabled = param->query_id_enabled; - query_id_const_merge = param->query_id_const_merge; + query_id_const_merge_threshold = param->query_id_const_merge_threshold; max_safe_fds = param->max_safe_fds; MaxBackends = param->MaxBackends; diff --git a/src/include/nodes/queryjumble.h b/src/include/nodes/queryjumble.h index 8ee2e9afbb6..a9f8cfcbed9 100644 --- a/src/include/nodes/queryjumble.h +++ b/src/include/nodes/queryjumble.h @@ -74,10 +74,10 @@ extern PGDLLIMPORT int compute_query_id; extern const char *CleanQuerytext(const char *query, int *location, int *len); extern JumbleState *JumbleQuery(Query *query); extern void EnableQueryId(void); -extern void SetQueryIdConstMerge(bool value); +extern void SetQueryIdConstMerge(int threshold); extern PGDLLIMPORT bool query_id_enabled; -extern PGDLLIMPORT bool query_id_const_merge; +extern PGDLLIMPORT int query_id_const_merge_threshold; /* * Returns whether query identifier computation has been enabled, either -- 2.41.0 --si27d3bgnnpsswlv-- ^ permalink raw reply [nested|flat] 8+ messages in thread
* [PATCH v18 4/4] Introduce query_id_const_merge_threshold @ 2024-01-22 20:31 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 8+ messages in thread From: Dmitrii Dolgov @ 2024-01-22 20:31 UTC (permalink / raw) Replace query_id_const_merge with a threshold to allow merging only if the number of elements is larger than specified value, which could be configured using pg_stat_statements parameter query_id_const_merge_threshold. --- .../pg_stat_statements/expected/merging.out | 68 ++++++++++++++++++- .../pg_stat_statements/pg_stat_statements.c | 36 +++++----- contrib/pg_stat_statements/sql/merging.sql | 21 +++++- doc/src/sgml/pgstatstatements.sgml | 23 ++++--- src/backend/nodes/queryjumblefuncs.c | 23 +++++-- src/backend/postmaster/postmaster.c | 6 +- src/include/nodes/queryjumble.h | 4 +- 7 files changed, 137 insertions(+), 44 deletions(-) diff --git a/contrib/pg_stat_statements/expected/merging.out b/contrib/pg_stat_statements/expected/merging.out index 0cb4f67b8b7..552e248ff14 100644 --- a/contrib/pg_stat_statements/expected/merging.out +++ b/contrib/pg_stat_statements/expected/merging.out @@ -36,7 +36,7 @@ SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; (4 rows) -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; t --- @@ -218,4 +218,68 @@ FROM cte; -------- (0 rows) -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +---------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4, $5, $6, $7, $8, $9) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 2 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(3 rows) + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +-------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(2 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 1 +(4 rows) + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index bb8c1894e05..6b438aeeeb9 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -264,8 +264,8 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static ProcessUtility_hook_type prev_ProcessUtility = NULL; -/* An assign hook to keep query_id_const_merge in sync */ -static void pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra); +/* An assign hook to keep query_id_const_merge_threshold in sync */ +static void pgss_query_id_const_merge_assign_hook(int newvalue, void *extra); /* Links to shared memory state */ static pgssSharedState *pgss = NULL; @@ -294,8 +294,8 @@ static bool pgss_track_utility = true; /* whether to track utility commands */ static bool pgss_track_planning = false; /* whether to track planning * duration */ static bool pgss_save = true; /* whether to save stats across shutdown */ -static bool pgss_query_id_const_merge = false; /* request constants merging - * when computing query_id */ +static int pgss_query_id_const_merge_threshold = 0; /* request constants merging + * when computing query_id */ #define pgss_enabled(level) \ (!IsParallelWorker() && \ @@ -458,20 +458,22 @@ _PG_init(void) NULL, NULL); - DefineCustomBoolVariable("pg_stat_statements.query_id_const_merge", - "Whether to merge constants in a list when computing query_id.", - NULL, - &pgss_query_id_const_merge, - false, - PGC_SUSET, - 0, - NULL, - pgss_query_id_const_merge_assign_hook, - NULL); + DefineCustomIntVariable("pg_stat_statements.query_id_const_merge_threshold", + "Whether to merge constants in a list when computing query_id.", + NULL, + &pgss_query_id_const_merge_threshold, + 0, + 0, + INT_MAX, + PGC_SUSET, + 0, + NULL, + pgss_query_id_const_merge_assign_hook, + NULL); MarkGUCPrefixReserved("pg_stat_statements"); - SetQueryIdConstMerge(pgss_query_id_const_merge); + SetQueryIdConstMerge(pgss_query_id_const_merge_threshold); /* * Install hooks. @@ -3068,10 +3070,10 @@ comp_location(const void *a, const void *b) } /* - * Notify query jumbling about query_id_const_merge status + * Notify query jumbling about query_id_const_merge_threshold status */ static void -pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra) +pgss_query_id_const_merge_assign_hook(int newvalue, void *extra) { SetQueryIdConstMerge(newvalue); } diff --git a/contrib/pg_stat_statements/sql/merging.sql b/contrib/pg_stat_statements/sql/merging.sql index 657044faded..fedeb35b8f5 100644 --- a/contrib/pg_stat_statements/sql/merging.sql +++ b/contrib/pg_stat_statements/sql/merging.sql @@ -15,7 +15,7 @@ SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; SELECT * FROM test_merge WHERE id IN (1); @@ -68,4 +68,21 @@ WITH cte AS ( SELECT ARRAY['a', 'b', 'c', const::varchar] AS result FROM cte; -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; + +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index bac029430be..3a060935bff 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -605,12 +605,12 @@ In some cases, queries with visibly different texts might get merged into a single <structname>pg_stat_statements</structname> entry. Normally this will happen only for semantically equivalent queries, or if - <varname>pg_stat_statements.query_id_const_merge</varname> is enabled and - the only difference between queries is the length of an array with constants - they contain: + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> is + enabled and the only difference between queries is the length of an array + with constants they contain: <screen> -=# SET query_id_const_merge = on; +=# SET query_id_const_merge_threshold = 1; =# SELECT pg_stat_statements_reset(); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); @@ -953,9 +953,9 @@ calls | 1 <varlistentry> <term> - <varname>pg_stat_statements.query_id_const_merge</varname> (<type>bool</type>) + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> (<type>integer</type>) <indexterm> - <primary><varname>pg_stat_statements.query_id_const_merge</varname> configuration parameter</primary> + <primary><varname>pg_stat_statements.query_id_const_merge_threshold</varname> configuration parameter</primary> </indexterm> </term> @@ -967,11 +967,12 @@ calls | 1 query will get multiple different identifiers, one for each occurrence with an array of different lenght. - If this parameter is on, an array of constants will contribute only the - first element, the last element and the number of elements to the query - identifier. It means two occurences of the same query, where the only - difference is number of constants in the array, are going to get the - same query identifier if the arrays are of similar length. + If this parameter is greater than 0, an array with more than + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> + constants will contribute only the first element, the last element + and the number of elements to the query identifier. It means two + occurences of the same query, where the only difference is number of + constants in the array, are going to get the same query identifier. Such queries are represented in form <literal>'(... [10-99 entries])'</literal>. The parameter could be used to reduce amount of repeating data stored diff --git a/src/backend/nodes/queryjumblefuncs.c b/src/backend/nodes/queryjumblefuncs.c index 7db7ae7f2ee..1884e91de49 100644 --- a/src/backend/nodes/queryjumblefuncs.c +++ b/src/backend/nodes/queryjumblefuncs.c @@ -44,8 +44,8 @@ /* GUC parameters */ int compute_query_id = COMPUTE_QUERY_ID_AUTO; -/* Whether to merge constants in a list when computing query_id */ -bool query_id_const_merge = false; +/* Lower threshold for the list length to merge constants when computing query_id */ +int query_id_const_merge_threshold = 1; /* True when compute_query_id is ON, or AUTO and a module requests them */ bool query_id_enabled = false; @@ -159,12 +159,14 @@ EnableQueryId(void) * Controls constants merging for query identifier computation. * * Third-party plugins can use this function to enable/disable merging - * of constants in a list when query identifier is computed. + * of constants in a list when query identifier is computed. The argument + * specifies the lower threshold for an array length, above which merging will + * be applied. */ void -SetQueryIdConstMerge(bool value) +SetQueryIdConstMerge(int threshold) { - query_id_const_merge = value; + query_id_const_merge_threshold = threshold; } /* @@ -240,7 +242,8 @@ RecordConstLocation(JumbleState *jstate, int location, int magnitude) /* * Verify if the provided list contains could be merged down, which means it - * contains only constant expressions. + * contains only constant expressions and the list contains more than + * query_id_const_merge_threshold elements. * * Return value is the order of magnitude (i.e. how many digits it has) for * length of the list (to use for representation purposes later on) if merging @@ -258,12 +261,18 @@ IsMergeableConstList(List *elements, Const **firstConst, Const **lastConst) if (elements == NULL) return 0; - if (!query_id_const_merge) + if (query_id_const_merge_threshold < 1) { /* Merging is disabled, process everything one by one */ return 0; } + if (elements->length < query_id_const_merge_threshold) + { + /* The list is not large enough */ + return 0; + } + firstExpr = linitial(elements); /* diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 367ea016fb3..78de378b4a9 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -533,7 +533,7 @@ typedef struct bool redirection_done; bool IsBinaryUpgrade; bool query_id_enabled; - bool query_id_const_merge; + int query_id_const_merge_threshold; int max_safe_fds; int MaxBackends; #ifdef WIN32 @@ -6113,7 +6113,7 @@ save_backend_variables(BackendParameters *param, Port *port, BackgroundWorker *w param->redirection_done = redirection_done; param->IsBinaryUpgrade = IsBinaryUpgrade; param->query_id_enabled = query_id_enabled; - param->query_id_const_merge = query_id_const_merge; + param->query_id_const_merge_threshold = query_id_const_merge_threshold; param->max_safe_fds = max_safe_fds; param->MaxBackends = MaxBackends; @@ -6359,7 +6359,7 @@ restore_backend_variables(BackendParameters *param, Port **port, BackgroundWorke redirection_done = param->redirection_done; IsBinaryUpgrade = param->IsBinaryUpgrade; query_id_enabled = param->query_id_enabled; - query_id_const_merge = param->query_id_const_merge; + query_id_const_merge_threshold = param->query_id_const_merge_threshold; max_safe_fds = param->max_safe_fds; MaxBackends = param->MaxBackends; diff --git a/src/include/nodes/queryjumble.h b/src/include/nodes/queryjumble.h index 551555494e0..ae5907aed84 100644 --- a/src/include/nodes/queryjumble.h +++ b/src/include/nodes/queryjumble.h @@ -74,10 +74,10 @@ extern PGDLLIMPORT int compute_query_id; extern const char *CleanQuerytext(const char *query, int *location, int *len); extern JumbleState *JumbleQuery(Query *query); extern void EnableQueryId(void); -extern void SetQueryIdConstMerge(bool value); +extern void SetQueryIdConstMerge(int threshold); extern PGDLLIMPORT bool query_id_enabled; -extern PGDLLIMPORT bool query_id_const_merge; +extern PGDLLIMPORT int query_id_const_merge_threshold; /* * Returns whether query identifier computation has been enabled, either -- 2.41.0 --tp75mykizkxh6a6a-- ^ permalink raw reply [nested|flat] 8+ messages in thread
* [PATCH v20 4/4] Introduce query_id_const_merge_threshold @ 2024-04-03 18:03 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 8+ messages in thread From: Dmitrii Dolgov @ 2024-04-03 18:03 UTC (permalink / raw) Replace query_id_const_merge with a threshold to allow merging only if the number of elements is larger than specified value, which could be configured using pg_stat_statements parameter query_id_const_merge_threshold. Reviewed-by: Sutou Kouhei Tested-by: Yasuo Honda --- .../pg_stat_statements/expected/merging.out | 68 ++++++++++++++++++- .../pg_stat_statements/pg_stat_statements.c | 36 +++++----- contrib/pg_stat_statements/sql/merging.sql | 21 +++++- doc/src/sgml/pgstatstatements.sgml | 23 ++++--- src/backend/nodes/queryjumblefuncs.c | 23 +++++-- src/backend/postmaster/launch_backend.c | 6 +- src/include/nodes/queryjumble.h | 4 +- 7 files changed, 137 insertions(+), 44 deletions(-) diff --git a/contrib/pg_stat_statements/expected/merging.out b/contrib/pg_stat_statements/expected/merging.out index 0cb4f67b8b7..552e248ff14 100644 --- a/contrib/pg_stat_statements/expected/merging.out +++ b/contrib/pg_stat_statements/expected/merging.out @@ -36,7 +36,7 @@ SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; (4 rows) -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; t --- @@ -218,4 +218,68 @@ FROM cte; -------- (0 rows) -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +---------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4, $5, $6, $7, $8, $9) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 2 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(3 rows) + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +-------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(2 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 1 +(4 rows) + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 0ade4cf515f..ffa832bbfb0 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -265,8 +265,8 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static ProcessUtility_hook_type prev_ProcessUtility = NULL; -/* An assign hook to keep query_id_const_merge in sync */ -static void pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra); +/* An assign hook to keep query_id_const_merge_threshold in sync */ +static void pgss_query_id_const_merge_assign_hook(int newvalue, void *extra); /* Links to shared memory state */ static pgssSharedState *pgss = NULL; @@ -295,8 +295,8 @@ static bool pgss_track_utility = true; /* whether to track utility commands */ static bool pgss_track_planning = false; /* whether to track planning * duration */ static bool pgss_save = true; /* whether to save stats across shutdown */ -static bool pgss_query_id_const_merge = false; /* request constants merging - * when computing query_id */ +static int pgss_query_id_const_merge_threshold = 0; /* request constants merging + * when computing query_id */ #define pgss_enabled(level) \ (!IsParallelWorker() && \ @@ -459,20 +459,22 @@ _PG_init(void) NULL, NULL); - DefineCustomBoolVariable("pg_stat_statements.query_id_const_merge", - "Whether to merge constants in a list when computing query_id.", - NULL, - &pgss_query_id_const_merge, - false, - PGC_SUSET, - 0, - NULL, - pgss_query_id_const_merge_assign_hook, - NULL); + DefineCustomIntVariable("pg_stat_statements.query_id_const_merge_threshold", + "Whether to merge constants in a list when computing query_id.", + NULL, + &pgss_query_id_const_merge_threshold, + 0, + 0, + INT_MAX, + PGC_SUSET, + 0, + NULL, + pgss_query_id_const_merge_assign_hook, + NULL); MarkGUCPrefixReserved("pg_stat_statements"); - SetQueryIdConstMerge(pgss_query_id_const_merge); + SetQueryIdConstMerge(pgss_query_id_const_merge_threshold); /* * Install hooks. @@ -3082,10 +3084,10 @@ comp_location(const void *a, const void *b) } /* - * Notify query jumbling about query_id_const_merge status + * Notify query jumbling about query_id_const_merge_threshold status */ static void -pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra) +pgss_query_id_const_merge_assign_hook(int newvalue, void *extra) { SetQueryIdConstMerge(newvalue); } diff --git a/contrib/pg_stat_statements/sql/merging.sql b/contrib/pg_stat_statements/sql/merging.sql index 657044faded..fedeb35b8f5 100644 --- a/contrib/pg_stat_statements/sql/merging.sql +++ b/contrib/pg_stat_statements/sql/merging.sql @@ -15,7 +15,7 @@ SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; SELECT * FROM test_merge WHERE id IN (1); @@ -68,4 +68,21 @@ WITH cte AS ( SELECT ARRAY['a', 'b', 'c', const::varchar] AS result FROM cte; -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; + +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index 12ffd021909..c939c316a37 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -605,12 +605,12 @@ In some cases, queries with visibly different texts might get merged into a single <structname>pg_stat_statements</structname> entry. Normally this will happen only for semantically equivalent queries, or if - <varname>pg_stat_statements.query_id_const_merge</varname> is enabled and - the only difference between queries is the length of an array with constants - they contain: + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> is + enabled and the only difference between queries is the length of an array + with constants they contain: <screen> -=# SET query_id_const_merge = on; +=# SET query_id_const_merge_threshold = 1; =# SELECT pg_stat_statements_reset(); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); @@ -959,9 +959,9 @@ calls | 1 <varlistentry> <term> - <varname>pg_stat_statements.query_id_const_merge</varname> (<type>bool</type>) + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> (<type>integer</type>) <indexterm> - <primary><varname>pg_stat_statements.query_id_const_merge</varname> configuration parameter</primary> + <primary><varname>pg_stat_statements.query_id_const_merge_threshold</varname> configuration parameter</primary> </indexterm> </term> @@ -973,11 +973,12 @@ calls | 1 query will get multiple different identifiers, one for each occurrence with an array of different lenght. - If this parameter is on, an array of constants will contribute only the - first element, the last element and the number of elements to the query - identifier. It means two occurences of the same query, where the only - difference is number of constants in the array, are going to get the - same query identifier if the arrays are of similar length. + If this parameter is greater than 0, an array with more than + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> + constants will contribute only the first element, the last element + and the number of elements to the query identifier. It means two + occurences of the same query, where the only difference is number of + constants in the array, are going to get the same query identifier. Such queries are represented in form <literal>'(... [10-99 entries])'</literal>. The parameter could be used to reduce amount of repeating data stored diff --git a/src/backend/nodes/queryjumblefuncs.c b/src/backend/nodes/queryjumblefuncs.c index e9ede0acb70..c25cb2be62a 100644 --- a/src/backend/nodes/queryjumblefuncs.c +++ b/src/backend/nodes/queryjumblefuncs.c @@ -44,8 +44,8 @@ /* GUC parameters */ int compute_query_id = COMPUTE_QUERY_ID_AUTO; -/* Whether to merge constants in a list when computing query_id */ -bool query_id_const_merge = false; +/* Lower threshold for the list length to merge constants when computing query_id */ +int query_id_const_merge_threshold = 1; /* * True when compute_query_id is ON or AUTO, and a module requests them. @@ -166,12 +166,14 @@ EnableQueryId(void) * Controls constants merging for query identifier computation. * * Third-party plugins can use this function to enable/disable merging - * of constants in a list when query identifier is computed. + * of constants in a list when query identifier is computed. The argument + * specifies the lower threshold for an array length, above which merging will + * be applied. */ void -SetQueryIdConstMerge(bool value) +SetQueryIdConstMerge(int threshold) { - query_id_const_merge = value; + query_id_const_merge_threshold = threshold; } /* @@ -249,7 +251,8 @@ RecordConstLocation(JumbleState *jstate, int location, int magnitude) /* * Verify if the provided list contains could be merged down, which means it - * contains only constant expressions. + * contains only constant expressions and the list contains more than + * query_id_const_merge_threshold elements. * * Return value is the order of magnitude (i.e. how many digits it has) for * length of the list (to use for representation purposes later on) if merging @@ -267,12 +270,18 @@ IsMergeableConstList(List *elements, Const **firstConst, Const **lastConst) if (elements == NIL) return 0; - if (!query_id_const_merge) + if (query_id_const_merge_threshold < 1) { /* Merging is disabled, process everything one by one */ return 0; } + if (elements->length < query_id_const_merge_threshold) + { + /* The list is not large enough */ + return 0; + } + firstExpr = linitial(elements); /* diff --git a/src/backend/postmaster/launch_backend.c b/src/backend/postmaster/launch_backend.c index 8bd73929183..491858a116e 100644 --- a/src/backend/postmaster/launch_backend.c +++ b/src/backend/postmaster/launch_backend.c @@ -122,7 +122,7 @@ typedef struct bool redirection_done; bool IsBinaryUpgrade; bool query_id_enabled; - bool query_id_const_merge; + int query_id_const_merge_threshold; int max_safe_fds; int MaxBackends; #ifdef WIN32 @@ -743,7 +743,7 @@ save_backend_variables(BackendParameters *param, ClientSocket *client_sock, param->redirection_done = redirection_done; param->IsBinaryUpgrade = IsBinaryUpgrade; param->query_id_enabled = query_id_enabled; - param->query_id_const_merge = query_id_const_merge; + param->query_id_const_merge_threshold = query_id_const_merge_threshold; param->max_safe_fds = max_safe_fds; param->MaxBackends = MaxBackends; @@ -1002,7 +1002,7 @@ restore_backend_variables(BackendParameters *param) redirection_done = param->redirection_done; IsBinaryUpgrade = param->IsBinaryUpgrade; query_id_enabled = param->query_id_enabled; - query_id_const_merge = param->query_id_const_merge; + query_id_const_merge_threshold = param->query_id_const_merge_threshold; max_safe_fds = param->max_safe_fds; MaxBackends = param->MaxBackends; diff --git a/src/include/nodes/queryjumble.h b/src/include/nodes/queryjumble.h index 0e69e420b7f..90218c60531 100644 --- a/src/include/nodes/queryjumble.h +++ b/src/include/nodes/queryjumble.h @@ -77,10 +77,10 @@ extern PGDLLIMPORT int compute_query_id; extern const char *CleanQuerytext(const char *query, int *location, int *len); extern JumbleState *JumbleQuery(Query *query); extern void EnableQueryId(void); -extern void SetQueryIdConstMerge(bool value); +extern void SetQueryIdConstMerge(int threshold); extern PGDLLIMPORT bool query_id_enabled; -extern PGDLLIMPORT bool query_id_const_merge; +extern PGDLLIMPORT int query_id_const_merge_threshold; /* * Returns whether query identifier computation has been enabled, either -- 2.41.0 --eqaczcvvorx3zguj-- ^ permalink raw reply [nested|flat] 8+ messages in thread
* [PATCH v19 4/4] Introduce query_id_const_merge_threshold @ 2024-04-03 18:03 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 8+ messages in thread From: Dmitrii Dolgov @ 2024-04-03 18:03 UTC (permalink / raw) Replace query_id_const_merge with a threshold to allow merging only if the number of elements is larger than specified value, which could be configured using pg_stat_statements parameter query_id_const_merge_threshold. --- .../pg_stat_statements/expected/merging.out | 68 ++++++++++++++++++- .../pg_stat_statements/pg_stat_statements.c | 36 +++++----- contrib/pg_stat_statements/sql/merging.sql | 21 +++++- doc/src/sgml/pgstatstatements.sgml | 23 ++++--- src/backend/nodes/queryjumblefuncs.c | 23 +++++-- src/backend/postmaster/launch_backend.c | 6 +- src/include/nodes/queryjumble.h | 4 +- 7 files changed, 137 insertions(+), 44 deletions(-) diff --git a/contrib/pg_stat_statements/expected/merging.out b/contrib/pg_stat_statements/expected/merging.out index 0cb4f67b8b7..552e248ff14 100644 --- a/contrib/pg_stat_statements/expected/merging.out +++ b/contrib/pg_stat_statements/expected/merging.out @@ -36,7 +36,7 @@ SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; (4 rows) -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; t --- @@ -218,4 +218,68 @@ FROM cte; -------- (0 rows) -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +---------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4, $5, $6, $7, $8, $9) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 2 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(3 rows) + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +-------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(2 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 1 +(4 rows) + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 00eec30feb1..569a74c4368 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -265,8 +265,8 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static ProcessUtility_hook_type prev_ProcessUtility = NULL; -/* An assign hook to keep query_id_const_merge in sync */ -static void pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra); +/* An assign hook to keep query_id_const_merge_threshold in sync */ +static void pgss_query_id_const_merge_assign_hook(int newvalue, void *extra); /* Links to shared memory state */ static pgssSharedState *pgss = NULL; @@ -295,8 +295,8 @@ static bool pgss_track_utility = true; /* whether to track utility commands */ static bool pgss_track_planning = false; /* whether to track planning * duration */ static bool pgss_save = true; /* whether to save stats across shutdown */ -static bool pgss_query_id_const_merge = false; /* request constants merging - * when computing query_id */ +static int pgss_query_id_const_merge_threshold = 0; /* request constants merging + * when computing query_id */ #define pgss_enabled(level) \ (!IsParallelWorker() && \ @@ -459,20 +459,22 @@ _PG_init(void) NULL, NULL); - DefineCustomBoolVariable("pg_stat_statements.query_id_const_merge", - "Whether to merge constants in a list when computing query_id.", - NULL, - &pgss_query_id_const_merge, - false, - PGC_SUSET, - 0, - NULL, - pgss_query_id_const_merge_assign_hook, - NULL); + DefineCustomIntVariable("pg_stat_statements.query_id_const_merge_threshold", + "Whether to merge constants in a list when computing query_id.", + NULL, + &pgss_query_id_const_merge_threshold, + 0, + 0, + INT_MAX, + PGC_SUSET, + 0, + NULL, + pgss_query_id_const_merge_assign_hook, + NULL); MarkGUCPrefixReserved("pg_stat_statements"); - SetQueryIdConstMerge(pgss_query_id_const_merge); + SetQueryIdConstMerge(pgss_query_id_const_merge_threshold); /* * Install hooks. @@ -3064,10 +3066,10 @@ comp_location(const void *a, const void *b) } /* - * Notify query jumbling about query_id_const_merge status + * Notify query jumbling about query_id_const_merge_threshold status */ static void -pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra) +pgss_query_id_const_merge_assign_hook(int newvalue, void *extra) { SetQueryIdConstMerge(newvalue); } diff --git a/contrib/pg_stat_statements/sql/merging.sql b/contrib/pg_stat_statements/sql/merging.sql index 657044faded..fedeb35b8f5 100644 --- a/contrib/pg_stat_statements/sql/merging.sql +++ b/contrib/pg_stat_statements/sql/merging.sql @@ -15,7 +15,7 @@ SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; SELECT * FROM test_merge WHERE id IN (1); @@ -68,4 +68,21 @@ WITH cte AS ( SELECT ARRAY['a', 'b', 'c', const::varchar] AS result FROM cte; -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; + +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index bac029430be..3a060935bff 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -605,12 +605,12 @@ In some cases, queries with visibly different texts might get merged into a single <structname>pg_stat_statements</structname> entry. Normally this will happen only for semantically equivalent queries, or if - <varname>pg_stat_statements.query_id_const_merge</varname> is enabled and - the only difference between queries is the length of an array with constants - they contain: + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> is + enabled and the only difference between queries is the length of an array + with constants they contain: <screen> -=# SET query_id_const_merge = on; +=# SET query_id_const_merge_threshold = 1; =# SELECT pg_stat_statements_reset(); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); @@ -953,9 +953,9 @@ calls | 1 <varlistentry> <term> - <varname>pg_stat_statements.query_id_const_merge</varname> (<type>bool</type>) + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> (<type>integer</type>) <indexterm> - <primary><varname>pg_stat_statements.query_id_const_merge</varname> configuration parameter</primary> + <primary><varname>pg_stat_statements.query_id_const_merge_threshold</varname> configuration parameter</primary> </indexterm> </term> @@ -967,11 +967,12 @@ calls | 1 query will get multiple different identifiers, one for each occurrence with an array of different lenght. - If this parameter is on, an array of constants will contribute only the - first element, the last element and the number of elements to the query - identifier. It means two occurences of the same query, where the only - difference is number of constants in the array, are going to get the - same query identifier if the arrays are of similar length. + If this parameter is greater than 0, an array with more than + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> + constants will contribute only the first element, the last element + and the number of elements to the query identifier. It means two + occurences of the same query, where the only difference is number of + constants in the array, are going to get the same query identifier. Such queries are represented in form <literal>'(... [10-99 entries])'</literal>. The parameter could be used to reduce amount of repeating data stored diff --git a/src/backend/nodes/queryjumblefuncs.c b/src/backend/nodes/queryjumblefuncs.c index a27202cb279..66696c87475 100644 --- a/src/backend/nodes/queryjumblefuncs.c +++ b/src/backend/nodes/queryjumblefuncs.c @@ -44,8 +44,8 @@ /* GUC parameters */ int compute_query_id = COMPUTE_QUERY_ID_AUTO; -/* Whether to merge constants in a list when computing query_id */ -bool query_id_const_merge = false; +/* Lower threshold for the list length to merge constants when computing query_id */ +int query_id_const_merge_threshold = 1; /* * True when compute_query_id is ON or AUTO, and a module requests them. @@ -165,12 +165,14 @@ EnableQueryId(void) * Controls constants merging for query identifier computation. * * Third-party plugins can use this function to enable/disable merging - * of constants in a list when query identifier is computed. + * of constants in a list when query identifier is computed. The argument + * specifies the lower threshold for an array length, above which merging will + * be applied. */ void -SetQueryIdConstMerge(bool value) +SetQueryIdConstMerge(int threshold) { - query_id_const_merge = value; + query_id_const_merge_threshold = threshold; } /* @@ -246,7 +248,8 @@ RecordConstLocation(JumbleState *jstate, int location, int magnitude) /* * Verify if the provided list contains could be merged down, which means it - * contains only constant expressions. + * contains only constant expressions and the list contains more than + * query_id_const_merge_threshold elements. * * Return value is the order of magnitude (i.e. how many digits it has) for * length of the list (to use for representation purposes later on) if merging @@ -264,12 +267,18 @@ IsMergeableConstList(List *elements, Const **firstConst, Const **lastConst) if (elements == NULL) return 0; - if (!query_id_const_merge) + if (query_id_const_merge_threshold < 1) { /* Merging is disabled, process everything one by one */ return 0; } + if (elements->length < query_id_const_merge_threshold) + { + /* The list is not large enough */ + return 0; + } + firstExpr = linitial(elements); /* diff --git a/src/backend/postmaster/launch_backend.c b/src/backend/postmaster/launch_backend.c index 5bb2f00c6ef..84b214a5952 100644 --- a/src/backend/postmaster/launch_backend.c +++ b/src/backend/postmaster/launch_backend.c @@ -122,7 +122,7 @@ typedef struct bool redirection_done; bool IsBinaryUpgrade; bool query_id_enabled; - bool query_id_const_merge; + int query_id_const_merge_threshold; int max_safe_fds; int MaxBackends; #ifdef WIN32 @@ -743,7 +743,7 @@ save_backend_variables(BackendParameters *param, ClientSocket *client_sock, param->redirection_done = redirection_done; param->IsBinaryUpgrade = IsBinaryUpgrade; param->query_id_enabled = query_id_enabled; - param->query_id_const_merge = query_id_const_merge; + param->query_id_const_merge_threshold = query_id_const_merge_threshold; param->max_safe_fds = max_safe_fds; param->MaxBackends = MaxBackends; @@ -1002,7 +1002,7 @@ restore_backend_variables(BackendParameters *param) redirection_done = param->redirection_done; IsBinaryUpgrade = param->IsBinaryUpgrade; query_id_enabled = param->query_id_enabled; - query_id_const_merge = param->query_id_const_merge; + query_id_const_merge_threshold = param->query_id_const_merge_threshold; max_safe_fds = param->max_safe_fds; MaxBackends = param->MaxBackends; diff --git a/src/include/nodes/queryjumble.h b/src/include/nodes/queryjumble.h index 551555494e0..ae5907aed84 100644 --- a/src/include/nodes/queryjumble.h +++ b/src/include/nodes/queryjumble.h @@ -74,10 +74,10 @@ extern PGDLLIMPORT int compute_query_id; extern const char *CleanQuerytext(const char *query, int *location, int *len); extern JumbleState *JumbleQuery(Query *query); extern void EnableQueryId(void); -extern void SetQueryIdConstMerge(bool value); +extern void SetQueryIdConstMerge(int threshold); extern PGDLLIMPORT bool query_id_enabled; -extern PGDLLIMPORT bool query_id_const_merge; +extern PGDLLIMPORT int query_id_const_merge_threshold; /* * Returns whether query identifier computation has been enabled, either -- 2.41.0 --yzrwxjftwx5vp4wd-- ^ permalink raw reply [nested|flat] 8+ messages in thread
* [PATCH v21 4/4] Introduce query_id_const_merge_threshold @ 2024-04-03 18:03 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 8+ messages in thread From: Dmitrii Dolgov @ 2024-04-03 18:03 UTC (permalink / raw) Replace query_id_const_merge with a threshold to allow merging only if the number of elements is larger than specified value, which could be configured using pg_stat_statements parameter query_id_const_merge_threshold. Reviewed-by: Sutou Kouhei Tested-by: Yasuo Honda --- .../pg_stat_statements/expected/merging.out | 68 ++++++++++++++++++- .../pg_stat_statements/pg_stat_statements.c | 36 +++++----- contrib/pg_stat_statements/sql/merging.sql | 21 +++++- doc/src/sgml/pgstatstatements.sgml | 23 ++++--- src/backend/nodes/queryjumblefuncs.c | 23 +++++-- src/backend/postmaster/launch_backend.c | 6 +- src/include/nodes/queryjumble.h | 4 +- 7 files changed, 137 insertions(+), 44 deletions(-) diff --git a/contrib/pg_stat_statements/expected/merging.out b/contrib/pg_stat_statements/expected/merging.out index 0cb4f67b8b..552e248ff1 100644 --- a/contrib/pg_stat_statements/expected/merging.out +++ b/contrib/pg_stat_statements/expected/merging.out @@ -36,7 +36,7 @@ SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; (4 rows) -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; t --- @@ -218,4 +218,68 @@ FROM cte; -------- (0 rows) -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +---------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4, $5, $6, $7, $8, $9) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 2 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(3 rows) + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +-------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(2 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 1 +(4 rows) + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 1c35e10117..ae672fcead 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -265,8 +265,8 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static ProcessUtility_hook_type prev_ProcessUtility = NULL; -/* An assign hook to keep query_id_const_merge in sync */ -static void pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra); +/* An assign hook to keep query_id_const_merge_threshold in sync */ +static void pgss_query_id_const_merge_assign_hook(int newvalue, void *extra); /* Links to shared memory state */ static pgssSharedState *pgss = NULL; @@ -295,8 +295,8 @@ static bool pgss_track_utility = true; /* whether to track utility commands */ static bool pgss_track_planning = false; /* whether to track planning * duration */ static bool pgss_save = true; /* whether to save stats across shutdown */ -static bool pgss_query_id_const_merge = false; /* request constants merging - * when computing query_id */ +static int pgss_query_id_const_merge_threshold = 0; /* request constants merging + * when computing query_id */ #define pgss_enabled(level) \ (!IsParallelWorker() && \ @@ -459,20 +459,22 @@ _PG_init(void) NULL, NULL); - DefineCustomBoolVariable("pg_stat_statements.query_id_const_merge", - "Whether to merge constants in a list when computing query_id.", - NULL, - &pgss_query_id_const_merge, - false, - PGC_SUSET, - 0, - NULL, - pgss_query_id_const_merge_assign_hook, - NULL); + DefineCustomIntVariable("pg_stat_statements.query_id_const_merge_threshold", + "Whether to merge constants in a list when computing query_id.", + NULL, + &pgss_query_id_const_merge_threshold, + 0, + 0, + INT_MAX, + PGC_SUSET, + 0, + NULL, + pgss_query_id_const_merge_assign_hook, + NULL); MarkGUCPrefixReserved("pg_stat_statements"); - SetQueryIdConstMerge(pgss_query_id_const_merge); + SetQueryIdConstMerge(pgss_query_id_const_merge_threshold); /* * Install hooks. @@ -3082,10 +3084,10 @@ comp_location(const void *a, const void *b) } /* - * Notify query jumbling about query_id_const_merge status + * Notify query jumbling about query_id_const_merge_threshold status */ static void -pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra) +pgss_query_id_const_merge_assign_hook(int newvalue, void *extra) { SetQueryIdConstMerge(newvalue); } diff --git a/contrib/pg_stat_statements/sql/merging.sql b/contrib/pg_stat_statements/sql/merging.sql index 657044fade..fedeb35b8f 100644 --- a/contrib/pg_stat_statements/sql/merging.sql +++ b/contrib/pg_stat_statements/sql/merging.sql @@ -15,7 +15,7 @@ SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; SELECT * FROM test_merge WHERE id IN (1); @@ -68,4 +68,21 @@ WITH cte AS ( SELECT ARRAY['a', 'b', 'c', const::varchar] AS result FROM cte; -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; + +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index 12ffd02190..c939c316a3 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -605,12 +605,12 @@ In some cases, queries with visibly different texts might get merged into a single <structname>pg_stat_statements</structname> entry. Normally this will happen only for semantically equivalent queries, or if - <varname>pg_stat_statements.query_id_const_merge</varname> is enabled and - the only difference between queries is the length of an array with constants - they contain: + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> is + enabled and the only difference between queries is the length of an array + with constants they contain: <screen> -=# SET query_id_const_merge = on; +=# SET query_id_const_merge_threshold = 1; =# SELECT pg_stat_statements_reset(); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); @@ -959,9 +959,9 @@ calls | 1 <varlistentry> <term> - <varname>pg_stat_statements.query_id_const_merge</varname> (<type>bool</type>) + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> (<type>integer</type>) <indexterm> - <primary><varname>pg_stat_statements.query_id_const_merge</varname> configuration parameter</primary> + <primary><varname>pg_stat_statements.query_id_const_merge_threshold</varname> configuration parameter</primary> </indexterm> </term> @@ -973,11 +973,12 @@ calls | 1 query will get multiple different identifiers, one for each occurrence with an array of different lenght. - If this parameter is on, an array of constants will contribute only the - first element, the last element and the number of elements to the query - identifier. It means two occurences of the same query, where the only - difference is number of constants in the array, are going to get the - same query identifier if the arrays are of similar length. + If this parameter is greater than 0, an array with more than + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> + constants will contribute only the first element, the last element + and the number of elements to the query identifier. It means two + occurences of the same query, where the only difference is number of + constants in the array, are going to get the same query identifier. Such queries are represented in form <literal>'(... [10-99 entries])'</literal>. The parameter could be used to reduce amount of repeating data stored diff --git a/src/backend/nodes/queryjumblefuncs.c b/src/backend/nodes/queryjumblefuncs.c index 1d3f36ca64..37a47072fb 100644 --- a/src/backend/nodes/queryjumblefuncs.c +++ b/src/backend/nodes/queryjumblefuncs.c @@ -44,8 +44,8 @@ /* GUC parameters */ int compute_query_id = COMPUTE_QUERY_ID_AUTO; -/* Whether to merge constants in a list when computing query_id */ -bool query_id_const_merge = false; +/* Lower threshold for the list length to merge constants when computing query_id */ +int query_id_const_merge_threshold = 1; /* * True when compute_query_id is ON or AUTO, and a module requests them. @@ -165,12 +165,14 @@ EnableQueryId(void) * Controls constants merging for query identifier computation. * * Third-party plugins can use this function to enable/disable merging - * of constants in a list when query identifier is computed. + * of constants in a list when query identifier is computed. The argument + * specifies the lower threshold for an array length, above which merging will + * be applied. */ void -SetQueryIdConstMerge(bool value) +SetQueryIdConstMerge(int threshold) { - query_id_const_merge = value; + query_id_const_merge_threshold = threshold; } /* @@ -248,7 +250,8 @@ RecordConstLocation(JumbleState *jstate, int location, int magnitude) /* * Verify if the provided list contains could be merged down, which means it - * contains only constant expressions. + * contains only constant expressions and the list contains more than + * query_id_const_merge_threshold elements. * * Return value is the order of magnitude (i.e. how many digits it has) for * length of the list (to use for representation purposes later on) if merging @@ -266,12 +269,18 @@ IsMergeableConstList(List *elements, Const **firstConst, Const **lastConst) if (elements == NIL) return 0; - if (!query_id_const_merge) + if (query_id_const_merge_threshold < 1) { /* Merging is disabled, process everything one by one */ return 0; } + if (elements->length < query_id_const_merge_threshold) + { + /* The list is not large enough */ + return 0; + } + firstExpr = linitial(elements); /* diff --git a/src/backend/postmaster/launch_backend.c b/src/backend/postmaster/launch_backend.c index da3ceceddb..f8a232b6a2 100644 --- a/src/backend/postmaster/launch_backend.c +++ b/src/backend/postmaster/launch_backend.c @@ -122,7 +122,7 @@ typedef struct bool redirection_done; bool IsBinaryUpgrade; bool query_id_enabled; - bool query_id_const_merge; + int query_id_const_merge_threshold; int max_safe_fds; int MaxBackends; #ifdef WIN32 @@ -731,7 +731,7 @@ save_backend_variables(BackendParameters *param, ClientSocket *client_sock, param->redirection_done = redirection_done; param->IsBinaryUpgrade = IsBinaryUpgrade; param->query_id_enabled = query_id_enabled; - param->query_id_const_merge = query_id_const_merge; + param->query_id_const_merge_threshold = query_id_const_merge_threshold; param->max_safe_fds = max_safe_fds; param->MaxBackends = MaxBackends; @@ -991,7 +991,7 @@ restore_backend_variables(BackendParameters *param) redirection_done = param->redirection_done; IsBinaryUpgrade = param->IsBinaryUpgrade; query_id_enabled = param->query_id_enabled; - query_id_const_merge = param->query_id_const_merge; + query_id_const_merge_threshold = param->query_id_const_merge_threshold; max_safe_fds = param->max_safe_fds; MaxBackends = param->MaxBackends; diff --git a/src/include/nodes/queryjumble.h b/src/include/nodes/queryjumble.h index 0e69e420b7..90218c6053 100644 --- a/src/include/nodes/queryjumble.h +++ b/src/include/nodes/queryjumble.h @@ -77,10 +77,10 @@ extern PGDLLIMPORT int compute_query_id; extern const char *CleanQuerytext(const char *query, int *location, int *len); extern JumbleState *JumbleQuery(Query *query); extern void EnableQueryId(void); -extern void SetQueryIdConstMerge(bool value); +extern void SetQueryIdConstMerge(int threshold); extern PGDLLIMPORT bool query_id_enabled; -extern PGDLLIMPORT bool query_id_const_merge; +extern PGDLLIMPORT int query_id_const_merge_threshold; /* * Returns whether query identifier computation has been enabled, either -- 2.45.1 --7t7bac4aexischdh-- ^ permalink raw reply [nested|flat] 8+ messages in thread
* [PATCH v22 4/4] Introduce query_id_const_merge_threshold @ 2024-04-03 18:03 Dmitrii Dolgov <9erthalion6@gmail.com> 0 siblings, 0 replies; 8+ messages in thread From: Dmitrii Dolgov @ 2024-04-03 18:03 UTC (permalink / raw) Replace query_id_const_merge with a threshold to allow merging only if the number of elements is larger than specified value, which could be configured using pg_stat_statements parameter query_id_const_merge_threshold. Reviewed-by: Sutou Kouhei Tested-by: Yasuo Honda --- .../pg_stat_statements/expected/merging.out | 68 ++++++++++++++++++- .../pg_stat_statements/pg_stat_statements.c | 36 +++++----- contrib/pg_stat_statements/sql/merging.sql | 21 +++++- doc/src/sgml/pgstatstatements.sgml | 23 ++++--- src/backend/nodes/queryjumblefuncs.c | 23 +++++-- src/backend/postmaster/launch_backend.c | 6 +- src/include/nodes/queryjumble.h | 4 +- 7 files changed, 137 insertions(+), 44 deletions(-) diff --git a/contrib/pg_stat_statements/expected/merging.out b/contrib/pg_stat_statements/expected/merging.out index 0cb4f67b8b..552e248ff1 100644 --- a/contrib/pg_stat_statements/expected/merging.out +++ b/contrib/pg_stat_statements/expected/merging.out @@ -36,7 +36,7 @@ SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; (4 rows) -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; t --- @@ -218,4 +218,68 @@ FROM cte; -------- (0 rows) -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); + id | data +----+------ +(0 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +---------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4, $5, $6, $7, $8, $9) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 2 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(3 rows) + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +-------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 +(2 rows) + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + id | data +----+------ +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT * FROM test_merge WHERE id IN ($1, $2, $3, $4) | 1 + SELECT * FROM test_merge WHERE id IN (... [10-99 entries]) | 1 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 1 +(4 rows) + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 24ab2a45ff..61b1c4ea30 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -267,8 +267,8 @@ static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static ProcessUtility_hook_type prev_ProcessUtility = NULL; -/* An assign hook to keep query_id_const_merge in sync */ -static void pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra); +/* An assign hook to keep query_id_const_merge_threshold in sync */ +static void pgss_query_id_const_merge_assign_hook(int newvalue, void *extra); /* Links to shared memory state */ static pgssSharedState *pgss = NULL; @@ -297,8 +297,8 @@ static bool pgss_track_utility = true; /* whether to track utility commands */ static bool pgss_track_planning = false; /* whether to track planning * duration */ static bool pgss_save = true; /* whether to save stats across shutdown */ -static bool pgss_query_id_const_merge = false; /* request constants merging - * when computing query_id */ +static int pgss_query_id_const_merge_threshold = 0; /* request constants merging + * when computing query_id */ #define pgss_enabled(level) \ (!IsParallelWorker() && \ @@ -463,20 +463,22 @@ _PG_init(void) NULL, NULL); - DefineCustomBoolVariable("pg_stat_statements.query_id_const_merge", - "Whether to merge constants in a list when computing query_id.", - NULL, - &pgss_query_id_const_merge, - false, - PGC_SUSET, - 0, - NULL, - pgss_query_id_const_merge_assign_hook, - NULL); + DefineCustomIntVariable("pg_stat_statements.query_id_const_merge_threshold", + "Whether to merge constants in a list when computing query_id.", + NULL, + &pgss_query_id_const_merge_threshold, + 0, + 0, + INT_MAX, + PGC_SUSET, + 0, + NULL, + pgss_query_id_const_merge_assign_hook, + NULL); MarkGUCPrefixReserved("pg_stat_statements"); - SetQueryIdConstMerge(pgss_query_id_const_merge); + SetQueryIdConstMerge(pgss_query_id_const_merge_threshold); /* * Install hooks. @@ -3084,10 +3086,10 @@ comp_location(const void *a, const void *b) } /* - * Notify query jumbling about query_id_const_merge status + * Notify query jumbling about query_id_const_merge_threshold status */ static void -pgss_query_id_const_merge_assign_hook(bool newvalue, void *extra) +pgss_query_id_const_merge_assign_hook(int newvalue, void *extra) { SetQueryIdConstMerge(newvalue); } diff --git a/contrib/pg_stat_statements/sql/merging.sql b/contrib/pg_stat_statements/sql/merging.sql index 657044fade..fedeb35b8f 100644 --- a/contrib/pg_stat_statements/sql/merging.sql +++ b/contrib/pg_stat_statements/sql/merging.sql @@ -15,7 +15,7 @@ SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; -- Normal scenario, too many simple constants for an IN query -SET pg_stat_statements.query_id_const_merge = on; +SET pg_stat_statements.query_id_const_merge_threshold = 1; SELECT pg_stat_statements_reset() IS NOT NULL AS t; SELECT * FROM test_merge WHERE id IN (1); @@ -68,4 +68,21 @@ WITH cte AS ( SELECT ARRAY['a', 'b', 'c', const::varchar] AS result FROM cte; -RESET pg_stat_statements.query_id_const_merge; +-- With the threshold +SET pg_stat_statements.query_id_const_merge_threshold = 10; + +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10); +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +-- With gaps on the threshold +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +SELECT * FROM test_merge WHERE id IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + +RESET pg_stat_statements.query_id_const_merge_threshold; diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index 55dfa5b50f..8d107e7706 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -623,12 +623,12 @@ In some cases, queries with visibly different texts might get merged into a single <structname>pg_stat_statements</structname> entry. Normally this will happen only for semantically equivalent queries, or if - <varname>pg_stat_statements.query_id_const_merge</varname> is enabled and - the only difference between queries is the length of an array with constants - they contain: + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> is + enabled and the only difference between queries is the length of an array + with constants they contain: <screen> -=# SET query_id_const_merge = on; +=# SET query_id_const_merge_threshold = 1; =# SELECT pg_stat_statements_reset(); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11); =# SELECT * FROM test WHERE a IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12); @@ -977,9 +977,9 @@ calls | 1 <varlistentry> <term> - <varname>pg_stat_statements.query_id_const_merge</varname> (<type>bool</type>) + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> (<type>integer</type>) <indexterm> - <primary><varname>pg_stat_statements.query_id_const_merge</varname> configuration parameter</primary> + <primary><varname>pg_stat_statements.query_id_const_merge_threshold</varname> configuration parameter</primary> </indexterm> </term> @@ -991,11 +991,12 @@ calls | 1 query will get multiple different identifiers, one for each occurrence with an array of different lenght. - If this parameter is on, an array of constants will contribute only the - first element, the last element and the number of elements to the query - identifier. It means two occurences of the same query, where the only - difference is number of constants in the array, are going to get the - same query identifier if the arrays are of similar length. + If this parameter is greater than 0, an array with more than + <varname>pg_stat_statements.query_id_const_merge_threshold</varname> + constants will contribute only the first element, the last element + and the number of elements to the query identifier. It means two + occurences of the same query, where the only difference is number of + constants in the array, are going to get the same query identifier. Such queries are represented in form <literal>'(... [10-99 entries])'</literal>. The parameter could be used to reduce amount of repeating data stored diff --git a/src/backend/nodes/queryjumblefuncs.c b/src/backend/nodes/queryjumblefuncs.c index 8ab3261d66..88a94be933 100644 --- a/src/backend/nodes/queryjumblefuncs.c +++ b/src/backend/nodes/queryjumblefuncs.c @@ -44,8 +44,8 @@ /* GUC parameters */ int compute_query_id = COMPUTE_QUERY_ID_AUTO; -/* Whether to merge constants in a list when computing query_id */ -bool query_id_const_merge = false; +/* Lower threshold for the list length to merge constants when computing query_id */ +int query_id_const_merge_threshold = 1; /* * True when compute_query_id is ON or AUTO, and a module requests them. @@ -172,12 +172,14 @@ EnableQueryId(void) * Controls constants merging for query identifier computation. * * Third-party plugins can use this function to enable/disable merging - * of constants in a list when query identifier is computed. + * of constants in a list when query identifier is computed. The argument + * specifies the lower threshold for an array length, above which merging will + * be applied. */ void -SetQueryIdConstMerge(bool value) +SetQueryIdConstMerge(int threshold) { - query_id_const_merge = value; + query_id_const_merge_threshold = threshold; } /* @@ -255,7 +257,8 @@ RecordConstLocation(JumbleState *jstate, int location, int magnitude) /* * Verify if the provided list contains could be merged down, which means it - * contains only constant expressions. + * contains only constant expressions and the list contains more than + * query_id_const_merge_threshold elements. * * Return value is the order of magnitude (i.e. how many digits it has) for * length of the list (to use for representation purposes later on) if merging @@ -273,12 +276,18 @@ IsMergeableConstList(List *elements, Const **firstConst, Const **lastConst) if (elements == NIL) return 0; - if (!query_id_const_merge) + if (query_id_const_merge_threshold < 1) { /* Merging is disabled, process everything one by one */ return 0; } + if (elements->length < query_id_const_merge_threshold) + { + /* The list is not large enough */ + return 0; + } + firstExpr = linitial(elements); /* diff --git a/src/backend/postmaster/launch_backend.c b/src/backend/postmaster/launch_backend.c index 0e2761edd5..8e438084e5 100644 --- a/src/backend/postmaster/launch_backend.c +++ b/src/backend/postmaster/launch_backend.c @@ -115,7 +115,7 @@ typedef struct bool redirection_done; bool IsBinaryUpgrade; bool query_id_enabled; - bool query_id_const_merge; + int query_id_const_merge_threshold; int max_safe_fds; int MaxBackends; int num_pmchild_slots; @@ -745,7 +745,7 @@ save_backend_variables(BackendParameters *param, param->redirection_done = redirection_done; param->IsBinaryUpgrade = IsBinaryUpgrade; param->query_id_enabled = query_id_enabled; - param->query_id_const_merge = query_id_const_merge; + param->query_id_const_merge_threshold = query_id_const_merge_threshold; param->max_safe_fds = max_safe_fds; param->MaxBackends = MaxBackends; @@ -1006,7 +1006,7 @@ restore_backend_variables(BackendParameters *param) redirection_done = param->redirection_done; IsBinaryUpgrade = param->IsBinaryUpgrade; query_id_enabled = param->query_id_enabled; - query_id_const_merge = param->query_id_const_merge; + query_id_const_merge_threshold = param->query_id_const_merge_threshold; max_safe_fds = param->max_safe_fds; MaxBackends = param->MaxBackends; diff --git a/src/include/nodes/queryjumble.h b/src/include/nodes/queryjumble.h index 0e69e420b7..90218c6053 100644 --- a/src/include/nodes/queryjumble.h +++ b/src/include/nodes/queryjumble.h @@ -77,10 +77,10 @@ extern PGDLLIMPORT int compute_query_id; extern const char *CleanQuerytext(const char *query, int *location, int *len); extern JumbleState *JumbleQuery(Query *query); extern void EnableQueryId(void); -extern void SetQueryIdConstMerge(bool value); +extern void SetQueryIdConstMerge(int threshold); extern PGDLLIMPORT bool query_id_enabled; -extern PGDLLIMPORT bool query_id_const_merge; +extern PGDLLIMPORT int query_id_const_merge_threshold; /* * Returns whether query identifier computation has been enabled, either -- 2.45.1 --nse6ci3hmyxhp7l3-- ^ permalink raw reply [nested|flat] 8+ messages in thread
end of thread, other threads:[~2024-04-03 18:03 UTC | newest] Thread overview: 8+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-02-02 00:57 [PATCH 7/9] Remove the special batch mode, use a larger buffer always Tomas Vondra <tomas.vondra@postgresql.org> 2023-10-16 14:52 [PATCH v16 4/4] Introduce query_id_const_merge_threshold Dmitrii Dolgov <9erthalion6@gmail.com> 2024-01-13 13:52 [PATCH v17 4/4] Introduce query_id_const_merge_threshold Dmitrii Dolgov <9erthalion6@gmail.com> 2024-01-22 20:31 [PATCH v18 4/4] Introduce query_id_const_merge_threshold Dmitrii Dolgov <9erthalion6@gmail.com> 2024-04-03 18:03 [PATCH v20 4/4] Introduce query_id_const_merge_threshold Dmitrii Dolgov <9erthalion6@gmail.com> 2024-04-03 18:03 [PATCH v19 4/4] Introduce query_id_const_merge_threshold Dmitrii Dolgov <9erthalion6@gmail.com> 2024-04-03 18:03 [PATCH v21 4/4] Introduce query_id_const_merge_threshold Dmitrii Dolgov <9erthalion6@gmail.com> 2024-04-03 18:03 [PATCH v22 4/4] Introduce query_id_const_merge_threshold Dmitrii Dolgov <9erthalion6@gmail.com>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox