public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 05/15] Maybe better than looping twice. For partitioned tables, create only "inherited" stats_ext. 21+ messages / 7 participants [nested] [flat]
* [PATCH 05/15] Maybe better than looping twice. For partitioned tables, create only "inherited" stats_ext. @ 2021-11-04 01:08 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Justin Pryzby @ 2021-11-04 01:08 UTC (permalink / raw) --- src/backend/commands/statscmds.c | 63 +++++--- src/backend/optimizer/util/plancat.c | 227 ++++++++++++++------------- 2 files changed, 151 insertions(+), 139 deletions(-) diff --git a/src/backend/commands/statscmds.c b/src/backend/commands/statscmds.c index 4395d878c7..89e9ad5843 100644 --- a/src/backend/commands/statscmds.c +++ b/src/backend/commands/statscmds.c @@ -45,6 +45,7 @@ static char *ChooseExtendedStatisticName(const char *name1, const char *name2, const char *label, Oid namespaceid); static char *ChooseExtendedStatisticNameAddition(List *exprs); +static void RemoveStatisticsByIdWorker(Relation pg_statistic_ext, Oid statsOid, bool inh); /* qsort comparator for the attnums in CreateStatistics */ @@ -524,8 +525,9 @@ CreateStatistics(CreateStatsStmt *stmt) datavalues[Anum_pg_statistic_ext_data_stxoid - 1] = ObjectIdGetDatum(statoid); - /* create only the "stxdinherit=false", because that always exists */ - datavalues[Anum_pg_statistic_ext_data_stxdinherit - 1] = ObjectIdGetDatum(false); + /* create "stxdinherit=false", because that always exists, except for + * partitioned tables, for which that's meaningless */ + datavalues[Anum_pg_statistic_ext_data_stxdinherit - 1] = BoolGetDatum(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); /* no statistics built yet */ datanulls[Anum_pg_statistic_ext_data_stxdndistinct - 1] = true; @@ -729,30 +731,7 @@ RemoveStatisticsById(Oid statsOid) HeapTuple tup; Form_pg_statistic_ext statext; Oid relid; - int inh; - - /* - * First delete the pg_statistic_ext_data tuple holding the actual - * statistical data. - */ - relation = table_open(StatisticExtDataRelationId, RowExclusiveLock); - - /* hack to delete both stxdinherit = true/false */ - for (inh = 0; inh <= 1; inh++) - { - tup = SearchSysCache2(STATEXTDATASTXOID, ObjectIdGetDatum(statsOid), - BoolGetDatum(inh)); - - if (!HeapTupleIsValid(tup)) /* should not happen */ - // elog(ERROR, "cache lookup failed for statistics data %u", statsOid); - continue; - - CatalogTupleDelete(relation, &tup->t_self); - - ReleaseSysCache(tup); - } - - table_close(relation, RowExclusiveLock); + char relkind; /* * Delete the pg_statistic_ext tuple. Also send out a cache inval on the @@ -767,6 +746,7 @@ RemoveStatisticsById(Oid statsOid) statext = (Form_pg_statistic_ext) GETSTRUCT(tup); relid = statext->stxrelid; + relkind = get_rel_relkind(relid); CacheInvalidateRelcacheByRelid(relid); @@ -775,6 +755,37 @@ RemoveStatisticsById(Oid statsOid) ReleaseSysCache(tup); table_close(relation, RowExclusiveLock); + + /* + * Delete the pg_statistic_ext_data tuples holding the actual + * statistical data. + */ + relation = table_open(StatisticExtDataRelationId, RowExclusiveLock); + + if (relkind != RELKIND_PARTITIONED_TABLE) + RemoveStatisticsByIdWorker(relation, statsOid, false); + RemoveStatisticsByIdWorker(relation, statsOid, true); + + table_close(relation, RowExclusiveLock); +} + +void +RemoveStatisticsByIdWorker(Relation pg_statistic_ext, Oid statsOid, bool inh) +{ + HeapTuple tup; + tup = SearchSysCache2(STATEXTDATASTXOID, ObjectIdGetDatum(statsOid), + BoolGetDatum(inh)); + + /* should not happen, .. */ + if (!HeapTupleIsValid(tup)) + { + if (!inh) + elog(ERROR, "cache lookup failed for statistics data %u inh %d", statsOid, inh); + return; + } + + CatalogTupleDelete(pg_statistic_ext, &tup->t_self); + ReleaseSysCache(tup); } /* diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 154d48a330..b1d8847529 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -81,6 +81,9 @@ static void set_baserel_partition_key_exprs(Relation relation, static void set_baserel_partition_constraint(Relation relation, RelOptInfo *rel); +static void get_relation_statistics_worker(RelOptInfo *rel, List **stainfos, + Oid statOid, HeapTuple htup, Form_pg_statistic_ext staForm, Index varno, + bool inh); /* * get_relation_info - @@ -1301,7 +1304,6 @@ get_relation_constraints(PlannerInfo *root, static List * get_relation_statistics(RelOptInfo *rel, Relation relation) { - Index varno = rel->relid; List *statoidlist; List *stainfos = NIL; ListCell *l; @@ -1312,150 +1314,149 @@ get_relation_statistics(RelOptInfo *rel, Relation relation) { Oid statOid = lfirst_oid(l); Form_pg_statistic_ext staForm; - Form_pg_statistic_ext_data dataForm; HeapTuple htup; - HeapTuple dtup; - Bitmapset *keys = NULL; - List *exprs = NIL; - int i; - int inh; htup = SearchSysCache1(STATEXTOID, ObjectIdGetDatum(statOid)); if (!HeapTupleIsValid(htup)) elog(ERROR, "cache lookup failed for statistics object %u", statOid); staForm = (Form_pg_statistic_ext) GETSTRUCT(htup); - /* - * Hack to load stats with stxdinherit true/false - there should be - * a better way to do this, I guess. - */ - for (inh = 0; inh <= 1; inh++) - { - dtup = SearchSysCache2(STATEXTDATASTXOID, - ObjectIdGetDatum(statOid), BoolGetDatum((bool) inh)); - if (!HeapTupleIsValid(dtup)) - continue; + get_relation_statistics_worker(rel, &stainfos, statOid, htup, staForm, rel->relid, false); + get_relation_statistics_worker(rel, &stainfos, statOid, htup, staForm, rel->relid, true); - dataForm = (Form_pg_statistic_ext_data) GETSTRUCT(dtup); + ReleaseSysCache(htup); + } - /* - * First, build the array of columns covered. This is ultimately - * wasted if no stats within the object have actually been built, but - * it doesn't seem worth troubling over that case. - */ - for (i = 0; i < staForm->stxkeys.dim1; i++) - keys = bms_add_member(keys, staForm->stxkeys.values[i]); + list_free(statoidlist); - /* - * Preprocess expressions (if any). We read the expressions, run them - * through eval_const_expressions, and fix the varnos. - */ - { - bool isnull; - Datum datum; + return stainfos; +} - /* decode expression (if any) */ - datum = SysCacheGetAttr(STATEXTOID, htup, - Anum_pg_statistic_ext_stxexprs, &isnull); +static void +get_relation_statistics_worker(RelOptInfo *rel, List **stainfos, Oid statOid, HeapTuple htup, Form_pg_statistic_ext staForm, Index varno, bool inh) +{ + Form_pg_statistic_ext_data dataForm; + HeapTuple dtup; + Bitmapset *keys = NULL; + List *exprs = NIL; + + dtup = SearchSysCache2(STATEXTDATASTXOID, + ObjectIdGetDatum(statOid), BoolGetDatum(inh)); + if (!HeapTupleIsValid(dtup)) + return; - if (!isnull) - { - char *exprsString; + dataForm = (Form_pg_statistic_ext_data) GETSTRUCT(dtup); - exprsString = TextDatumGetCString(datum); - exprs = (List *) stringToNode(exprsString); - pfree(exprsString); + /* + * First, build the array of columns covered. This is ultimately + * wasted if no stats within the object have actually been built, but + * it doesn't seem worth troubling over that case. + */ + for (int i = 0; i < staForm->stxkeys.dim1; i++) + keys = bms_add_member(keys, staForm->stxkeys.values[i]); - /* - * Run the expressions through eval_const_expressions. This is - * not just an optimization, but is necessary, because the - * planner will be comparing them to similarly-processed qual - * clauses, and may fail to detect valid matches without this. - * We must not use canonicalize_qual, however, since these - * aren't qual expressions. - */ - exprs = (List *) eval_const_expressions(NULL, (Node *) exprs); + /* + * Preprocess expressions (if any). We read the expressions, run them + * through eval_const_expressions, and fix the varnos. + */ + { + bool isnull; + Datum datum; - /* May as well fix opfuncids too */ - fix_opfuncids((Node *) exprs); + /* decode expression (if any) */ + datum = SysCacheGetAttr(STATEXTOID, htup, + Anum_pg_statistic_ext_stxexprs, &isnull); - /* - * Modify the copies we obtain from the relcache to have the - * correct varno for the parent relation, so that they match - * up correctly against qual clauses. - */ - if (varno != 1) - ChangeVarNodes((Node *) exprs, 1, varno, 0); - } - } - - /* add one StatisticExtInfo for each kind built */ - if (statext_is_kind_built(dtup, STATS_EXT_NDISTINCT)) - { - StatisticExtInfo *info = makeNode(StatisticExtInfo); + if (!isnull) + { + char *exprsString; - info->statOid = statOid; - info->inherit = dataForm->stxdinherit; - info->rel = rel; - info->kind = STATS_EXT_NDISTINCT; - info->keys = bms_copy(keys); - info->exprs = exprs; + exprsString = TextDatumGetCString(datum); + exprs = (List *) stringToNode(exprsString); + pfree(exprsString); - stainfos = lappend(stainfos, info); - } + /* + * Run the expressions through eval_const_expressions. This is + * not just an optimization, but is necessary, because the + * planner will be comparing them to similarly-processed qual + * clauses, and may fail to detect valid matches without this. + * We must not use canonicalize_qual, however, since these + * aren't qual expressions. + */ + exprs = (List *) eval_const_expressions(NULL, (Node *) exprs); - if (statext_is_kind_built(dtup, STATS_EXT_DEPENDENCIES)) - { - StatisticExtInfo *info = makeNode(StatisticExtInfo); + /* May as well fix opfuncids too */ + fix_opfuncids((Node *) exprs); - info->statOid = statOid; - info->inherit = dataForm->stxdinherit; - info->rel = rel; - info->kind = STATS_EXT_DEPENDENCIES; - info->keys = bms_copy(keys); - info->exprs = exprs; + /* + * Modify the copies we obtain from the relcache to have the + * correct varno for the parent relation, so that they match + * up correctly against qual clauses. + */ + if (varno != 1) + ChangeVarNodes((Node *) exprs, 1, varno, 0); + } + } - stainfos = lappend(stainfos, info); - } + /* add one StatisticExtInfo for each kind built */ + if (statext_is_kind_built(dtup, STATS_EXT_NDISTINCT)) + { + StatisticExtInfo *info = makeNode(StatisticExtInfo); - if (statext_is_kind_built(dtup, STATS_EXT_MCV)) - { - StatisticExtInfo *info = makeNode(StatisticExtInfo); + info->statOid = statOid; + info->inherit = dataForm->stxdinherit; + info->rel = rel; + info->kind = STATS_EXT_NDISTINCT; + info->keys = bms_copy(keys); + info->exprs = exprs; - info->statOid = statOid; - info->inherit = dataForm->stxdinherit; - info->rel = rel; - info->kind = STATS_EXT_MCV; - info->keys = bms_copy(keys); - info->exprs = exprs; + *stainfos = lappend(*stainfos, info); + } - stainfos = lappend(stainfos, info); - } + if (statext_is_kind_built(dtup, STATS_EXT_DEPENDENCIES)) + { + StatisticExtInfo *info = makeNode(StatisticExtInfo); - if (statext_is_kind_built(dtup, STATS_EXT_EXPRESSIONS)) - { - StatisticExtInfo *info = makeNode(StatisticExtInfo); + info->statOid = statOid; + info->inherit = dataForm->stxdinherit; + info->rel = rel; + info->kind = STATS_EXT_DEPENDENCIES; + info->keys = bms_copy(keys); + info->exprs = exprs; - info->statOid = statOid; - info->inherit = dataForm->stxdinherit; - info->rel = rel; - info->kind = STATS_EXT_EXPRESSIONS; - info->keys = bms_copy(keys); - info->exprs = exprs; + *stainfos = lappend(*stainfos, info); + } - stainfos = lappend(stainfos, info); - } + if (statext_is_kind_built(dtup, STATS_EXT_MCV)) + { + StatisticExtInfo *info = makeNode(StatisticExtInfo); - ReleaseSysCache(dtup); - } + info->statOid = statOid; + info->inherit = dataForm->stxdinherit; + info->rel = rel; + info->kind = STATS_EXT_MCV; + info->keys = bms_copy(keys); + info->exprs = exprs; - ReleaseSysCache(htup); - bms_free(keys); + *stainfos = lappend(*stainfos, info); } - list_free(statoidlist); + if (statext_is_kind_built(dtup, STATS_EXT_EXPRESSIONS)) + { + StatisticExtInfo *info = makeNode(StatisticExtInfo); - return stainfos; + info->statOid = statOid; + info->inherit = dataForm->stxdinherit; + info->rel = rel; + info->kind = STATS_EXT_EXPRESSIONS; + info->keys = bms_copy(keys); + info->exprs = exprs; + + *stainfos = lappend(*stainfos, info); + } + + ReleaseSysCache(dtup); + bms_free(keys); } /* -- 2.17.0 --qmeTShRBchLQhVIG Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Refactor-parent-ACL-check.patch" ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2023-11-22 05:32 Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2023-11-22 05:32 UTC (permalink / raw) To: [email protected]; Andrei Lepikhov <[email protected]>; +Cc: Ashutosh Bapat <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hello Alena, Andrei, and all, Thank you for reviewing this patch. I really apologize for not updating this thread for a while. On Sat, Nov 18, 2023 at 6:04 AM Alena Rybakina <[email protected]> wrote: > Hi, all! > > While I was reviewing the patches, I noticed that they needed some rebasing, and in one of the patches (Introduce-indexes-for-RestrictInfo.patch) there was a conflict with the recently added self-join-removal feature [1]. So, I rebased patches and resolved the conflicts. While I was doing this, I found a problem that I also fixed: Thank you very much for rebasing these patches and fixing the issue. The bug seemed to be caused because these indexes were in RangeTblEntry, and the handling of their serialization was not correct. Thank you for fixing it. On Mon, Nov 20, 2023 at 1:45 PM Andrei Lepikhov <[email protected]> wrote: > During the work on committing the SJE feature [1], Alexander Korotkov > pointed out the silver lining in this work [2]: he proposed that we > shouldn't remove RelOptInfo from simple_rel_array at all but replace it > with an 'Alias', which will refer the kept relation. It can simplify > further optimizations on removing redundant parts of the query. Thank you for sharing this information. I think the idea suggested by Alexander Korotkov is also helpful for our patch. As mentioned above, the indexes are in RangeTblEntry in the current implementation. However, I think RangeTblEntry is not the best place to store them. An 'alias' relids may help solve this and simplify fixing the above bug. I will try this approach soon. Unfortunately, I've been busy due to work, so I won't be able to respond for several weeks. I'm really sorry for not being able to see the patches. As soon as I'm not busy, I will look at them, consider the above approach, and reply to this thread. If there is no objection, I will move this CF entry forward to next CF. Again, thank you very much for looking at this thread, and I'm sorry for my late work. -- Best regards, Yuya Watari ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2023-11-30 04:18 Yuya Watari <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2023-11-30 04:18 UTC (permalink / raw) To: [email protected]; Andrei Lepikhov <[email protected]>; +Cc: Ashutosh Bapat <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hello, On Wed, Nov 22, 2023 at 2:32 PM Yuya Watari <[email protected]> wrote: > Unfortunately, I've been busy due to work, so I won't be able to > respond for several weeks. I'm really sorry for not being able to see > the patches. As soon as I'm not busy, I will look at them, consider > the above approach, and reply to this thread. If there is no > objection, I will move this CF entry forward to next CF. Since the end of this month is approaching, I moved this CF entry to the next CF (January CF). I will reply to this thread in a few weeks. Again, I appreciate your kind reviews and patches. -- Best regards, Yuya Watari ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2023-12-13 06:21 Yuya Watari <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2023-12-13 06:21 UTC (permalink / raw) To: [email protected]; Andrei Lepikhov <[email protected]>; +Cc: Ashutosh Bapat <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hello Alena, Andrei, and all, I am sorry for my late response. I found that the current patches do not apply to the master, so I have rebased those patches. I have attached v22. For this later discussion, I separated the rebasing and bug fixing that Alena did in v21 into separate commits, v22-0003 and v22-0004. I will merge these commits after the discussion. 1. v22-0003 (solved_conflict_with_self_join_removal.txt) Thank you for your rebase. Looking at your rebasing patch, I thought we could do this more simply. Your patch deletes (more precisely, sets to null) non-redundant members from the root->eq_sources list and re-adds them to the same list. However, this approach seems a little waste of memory. Instead, we can update EquivalenceClass->ec_source_indexes directly. Then, we can reuse the members in root->eq_sources and don't need to extend root->eq_sources. I did this in v22-0003. What do you think of this approach? The main concern with this idea is that it does not fix RangeTblEntry->eclass_source_indexes. The current code works fine even if we don't fix the index because get_ec_source_indexes() always does bms_intersect() for eclass_source_indexes and ec_source_indexes. If we guaranteed this behavior of doing bms_intersect, then simply modifying ec_source_indexes would be fine. Fortunately, such a guarantee is not so difficult. And your patch removes the following assertion code from the previous patch. May I ask why you removed this code? I think this assertion is helpful for sanity checks. Of course, I know that this kind of assertion will slow down regression tests or assert-enabled builds. So, we may have to discuss which assertions to keep and which to discard. ===== -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(rel_esis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, - i); - - Assert(bms_overlap(relids, rinfo->clause_relids)); - } -#endif ===== Finally, your patch changes the name of the following function. I understand the need for this change, but it has nothing to do with our patches, so we should not include it and discuss it in another thread. ===== -update_eclasses(EquivalenceClass *ec, int from, int to) +update_eclass(PlannerInfo *root, EquivalenceClass *ec, int from, int to) ===== 2. v22-0004 (bug_related_to_atomic_function.txt) Thank you for fixing the bug. As I wrote in the previous mail: On Wed, Nov 22, 2023 at 2:32 PM Yuya Watari <[email protected]> wrote: > On Mon, Nov 20, 2023 at 1:45 PM Andrei Lepikhov > <[email protected]> wrote: > > During the work on committing the SJE feature [1], Alexander Korotkov > > pointed out the silver lining in this work [2]: he proposed that we > > shouldn't remove RelOptInfo from simple_rel_array at all but replace it > > with an 'Alias', which will refer the kept relation. It can simplify > > further optimizations on removing redundant parts of the query. > > Thank you for sharing this information. I think the idea suggested by > Alexander Korotkov is also helpful for our patch. As mentioned above, > the indexes are in RangeTblEntry in the current implementation. > However, I think RangeTblEntry is not the best place to store them. An > 'alias' relids may help solve this and simplify fixing the above bug. > I will try this approach soon. I think that the best way to solve this issue is to move these indexes from RangeTblEntry to RelOptInfo. Since they are related to planning time, they should be in RelOptInfo. The reason why I put these indexes in RangeTblEntry is because some RelOptInfos can be null and we cannot store the indexes. This problem is similar to an issue regarding 'varno 0' Vars. I hope an alias RelOptInfo would help solve this issue. I have attached the current proof of concept I am considering as poc-alias-reloptinfo.txt. To test this patch, please follow the procedure below. 1. Apply all *.patch files, 2. Apply Alexander Korotkov's alias_relids.patch [1], and 3. Apply poc-alias-reloptinfo.txt, which is attached to this email. My patch creates a dummy (or an alias) RelOptInfo to store indexes if the corresponding RelOptInfo is null. The following is the core change in my patch. ===== @@ -627,9 +627,19 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + /* + * If the corresponding RelOptInfo does not exist, we create a 'dummy' + * RelOptInfo for storing EquivalenceClass indexes. + */ + if (rel == NULL) + { + rel = root->simple_rel_array[i] = makeNode(RelOptInfo); + rel->eclass_source_indexes = NULL; + rel->eclass_derive_indexes = NULL; + } + rel->eclass_source_indexes = bms_add_member(rel->eclass_source_indexes, source_idx); } ===== At this point, I'm not sure if this approach is correct. It seems to pass the regression tests, but we should doubt its correctness. I will continue to experiment with this idea. [1] https://www.postgresql.org/message-id/CAPpHfdseB13zJJPZuBORevRnZ0vcFyUaaJeSGfAysX7S5er%2BEQ%40mail.g... -- Best regards, Yuya Watari diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 1904a53acb..241be97920 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -569,8 +569,6 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(inh); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); - WRITE_BITMAPSET_FIELD(eclass_source_indexes); - WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c index 314736d989..813eda3e73 100644 --- a/src/backend/nodes/read.c +++ b/src/backend/nodes/read.c @@ -514,23 +514,3 @@ nodeRead(const char *token, int tok_len) return (void *) result; } - -/* - * pg_strtok_save_context - - * Save context initialized by stringToNode. - */ -void -pg_strtok_save_context(const char **pcontext) -{ - *pcontext = pg_strtok_ptr; -} - -/* - * pg_strtok_restore_context - - * Resore saved context. - */ -void -pg_strtok_restore_context(const char *context) -{ - pg_strtok_ptr = context; -} diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 0915c0e661..cc2021c1f7 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -191,26 +191,6 @@ nullable_string(const char *token, int length) return debackslash(token, length); } -/* Read an equivalence field (anything written as ":fldname %u") and check it */ -#define READ_EQ_BITMAPSET_FIELD_CHECK(fldname) \ -{ \ - int save_length = length; \ - const char *context; \ - pg_strtok_save_context(&context); \ - token = pg_strtok(&length); \ - if (length > 0 && strncmp(token, ":"#fldname, strlen(":"#fldname))) \ - { \ - /* "fldname" field was not found - fill it and restore context. */ \ - local_node->fldname = NULL; \ - pg_strtok_restore_context(context); \ - length = save_length; \ - } \ - else \ - { \ - local_node->fldname = _readBitmapset(); \ - } \ -} - /* * _readBitmapset @@ -594,8 +574,6 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_EQ_BITMAPSET_FIELD_CHECK(eclass_source_indexes); - READ_EQ_BITMAPSET_FIELD_CHECK(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 813a365e63..7e0a0c45f1 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -627,9 +627,19 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + /* + * If the corresponding RelOptInfo does not exist, we create a 'dummy' + * RelOptInfo for storing EquivalenceClass indexes. + */ + if (rel == NULL) + { + rel = root->simple_rel_array[i] = makeNode(RelOptInfo); + rel->eclass_source_indexes = NULL; + rel->eclass_derive_indexes = NULL; + } + rel->eclass_source_indexes = bms_add_member(rel->eclass_source_indexes, source_idx); } } @@ -649,9 +659,19 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + /* + * If the corresponding RelOptInfo does not exist, we create a 'dummy' + * RelOptInfo for storing EquivalenceClass indexes. + */ + if (rel == NULL) + { + rel = root->simple_rel_array[i] = makeNode(RelOptInfo); + rel->eclass_source_indexes = NULL; + rel->eclass_derive_indexes = NULL; + } + rel->eclass_derive_indexes = bms_add_member(rel->eclass_derive_indexes, derive_idx); } } @@ -3667,9 +3687,9 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + rel_esis = bms_add_members(rel_esis, rel->eclass_source_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3705,7 +3725,7 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3714,12 +3734,12 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ esis = bms_intersect(ec->ec_source_indexes, - rte->eclass_source_indexes); + rel->eclass_source_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - esis = bms_int_members(esis, rte->eclass_source_indexes); + rel = root->simple_rel_array[i]; + esis = bms_int_members(esis, rel->eclass_source_indexes); } } @@ -3756,9 +3776,9 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + rel_edis = bms_add_members(rel_edis, rel->eclass_derive_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3794,7 +3814,7 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3803,12 +3823,12 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, - rte->eclass_derive_indexes); + rel->eclass_derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - edis = bms_int_members(edis, rte->eclass_derive_indexes); + rel = root->simple_rel_array[i]; + edis = bms_int_members(edis, rel->eclass_derive_indexes); } } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index b75e92b2e1..d826f072c7 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -483,13 +483,6 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); - /* - * We do not want to inherit the EquivalenceMember indexes of the parent - * to its child - */ - childrte->eclass_source_indexes = NULL; - childrte->eclass_derive_indexes = NULL; - Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 698ddfd67c..d3bd24bbb3 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -264,6 +264,8 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->tuples = 0; rel->allvisfrac = 0; rel->eclass_indexes = NULL; + rel->eclass_source_indexes = NULL; + rel->eclass_derive_indexes = NULL; rel->subroot = NULL; rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ @@ -757,6 +759,8 @@ build_join_rel(PlannerInfo *root, joinrel->tuples = 0; joinrel->allvisfrac = 0; joinrel->eclass_indexes = NULL; + joinrel->eclass_source_indexes = NULL; + joinrel->eclass_derive_indexes = NULL; joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; @@ -955,6 +959,8 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->tuples = 0; joinrel->allvisfrac = 0; joinrel->eclass_indexes = NULL; + joinrel->eclass_source_indexes = NULL; + joinrel->eclass_derive_indexes = NULL; joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 475a65475e..e494309da8 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1194,12 +1194,6 @@ typedef struct RangeTblEntry bool inh; /* inheritance requested? */ bool inFromCl; /* present in FROM clause? */ List *securityQuals; /* security barrier quals to apply, if any */ - Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources - * list for RestrictInfos that mention - * this relation */ - Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives - * list for RestrictInfos that mention - * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 7c94ece332..5571585355 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -943,6 +943,19 @@ typedef struct RelOptInfo double allvisfrac; /* indexes in PlannerInfo's eq_classes list of ECs that mention this rel */ Bitmapset *eclass_indexes; + + /* + * Indexes in PlannerInfo's eq_sources list for RestrictInfos that mention + * this relation. + */ + Bitmapset *eclass_source_indexes; + + /* + * Indexes in PlannerInfo's eq_derives list for RestrictInfos that mention + * this relation. + */ + Bitmapset *eclass_derive_indexes; + PlannerInfo *subroot; /* if subquery */ List *subplan_params; /* if subquery */ /* wanted number of parallel workers */ diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h index 2fa68e59cd..cba6f0be75 100644 --- a/src/include/nodes/readfuncs.h +++ b/src/include/nodes/readfuncs.h @@ -29,8 +29,6 @@ extern PGDLLIMPORT bool restore_location_fields; extern const char *pg_strtok(int *length); extern char *debackslash(const char *token, int length); extern void *nodeRead(const char *token, int tok_len); -extern void pg_strtok_save_context(const char **pcontext); -extern void pg_strtok_restore_context(const char *context); /* * prototypes for functions in readfuncs.c Attachments: [text/plain] poc-alias-reloptinfo.txt (10.3K, ../../CAJ2pMkZGB4Yx1dCYkU_YRJgj2rcC0s+PWknqrszmsRPHGLqCgg@mail.gmail.com/2-poc-alias-reloptinfo.txt) download | inline diff: diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 1904a53acb..241be97920 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -569,8 +569,6 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(inh); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); - WRITE_BITMAPSET_FIELD(eclass_source_indexes); - WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c index 314736d989..813eda3e73 100644 --- a/src/backend/nodes/read.c +++ b/src/backend/nodes/read.c @@ -514,23 +514,3 @@ nodeRead(const char *token, int tok_len) return (void *) result; } - -/* - * pg_strtok_save_context - - * Save context initialized by stringToNode. - */ -void -pg_strtok_save_context(const char **pcontext) -{ - *pcontext = pg_strtok_ptr; -} - -/* - * pg_strtok_restore_context - - * Resore saved context. - */ -void -pg_strtok_restore_context(const char *context) -{ - pg_strtok_ptr = context; -} diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 0915c0e661..cc2021c1f7 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -191,26 +191,6 @@ nullable_string(const char *token, int length) return debackslash(token, length); } -/* Read an equivalence field (anything written as ":fldname %u") and check it */ -#define READ_EQ_BITMAPSET_FIELD_CHECK(fldname) \ -{ \ - int save_length = length; \ - const char *context; \ - pg_strtok_save_context(&context); \ - token = pg_strtok(&length); \ - if (length > 0 && strncmp(token, ":"#fldname, strlen(":"#fldname))) \ - { \ - /* "fldname" field was not found - fill it and restore context. */ \ - local_node->fldname = NULL; \ - pg_strtok_restore_context(context); \ - length = save_length; \ - } \ - else \ - { \ - local_node->fldname = _readBitmapset(); \ - } \ -} - /* * _readBitmapset @@ -594,8 +574,6 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_EQ_BITMAPSET_FIELD_CHECK(eclass_source_indexes); - READ_EQ_BITMAPSET_FIELD_CHECK(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 813a365e63..7e0a0c45f1 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -627,9 +627,19 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + /* + * If the corresponding RelOptInfo does not exist, we create a 'dummy' + * RelOptInfo for storing EquivalenceClass indexes. + */ + if (rel == NULL) + { + rel = root->simple_rel_array[i] = makeNode(RelOptInfo); + rel->eclass_source_indexes = NULL; + rel->eclass_derive_indexes = NULL; + } + rel->eclass_source_indexes = bms_add_member(rel->eclass_source_indexes, source_idx); } } @@ -649,9 +659,19 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + /* + * If the corresponding RelOptInfo does not exist, we create a 'dummy' + * RelOptInfo for storing EquivalenceClass indexes. + */ + if (rel == NULL) + { + rel = root->simple_rel_array[i] = makeNode(RelOptInfo); + rel->eclass_source_indexes = NULL; + rel->eclass_derive_indexes = NULL; + } + rel->eclass_derive_indexes = bms_add_member(rel->eclass_derive_indexes, derive_idx); } } @@ -3667,9 +3687,9 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + rel_esis = bms_add_members(rel_esis, rel->eclass_source_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3705,7 +3725,7 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3714,12 +3734,12 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ esis = bms_intersect(ec->ec_source_indexes, - rte->eclass_source_indexes); + rel->eclass_source_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - esis = bms_int_members(esis, rte->eclass_source_indexes); + rel = root->simple_rel_array[i]; + esis = bms_int_members(esis, rel->eclass_source_indexes); } } @@ -3756,9 +3776,9 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; - rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + rel_edis = bms_add_members(rel_edis, rel->eclass_derive_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3794,7 +3814,7 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + RelOptInfo *rel = root->simple_rel_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3803,12 +3823,12 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, - rte->eclass_derive_indexes); + rel->eclass_derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - edis = bms_int_members(edis, rte->eclass_derive_indexes); + rel = root->simple_rel_array[i]; + edis = bms_int_members(edis, rel->eclass_derive_indexes); } } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index b75e92b2e1..d826f072c7 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -483,13 +483,6 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); - /* - * We do not want to inherit the EquivalenceMember indexes of the parent - * to its child - */ - childrte->eclass_source_indexes = NULL; - childrte->eclass_derive_indexes = NULL; - Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 698ddfd67c..d3bd24bbb3 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -264,6 +264,8 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->tuples = 0; rel->allvisfrac = 0; rel->eclass_indexes = NULL; + rel->eclass_source_indexes = NULL; + rel->eclass_derive_indexes = NULL; rel->subroot = NULL; rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ @@ -757,6 +759,8 @@ build_join_rel(PlannerInfo *root, joinrel->tuples = 0; joinrel->allvisfrac = 0; joinrel->eclass_indexes = NULL; + joinrel->eclass_source_indexes = NULL; + joinrel->eclass_derive_indexes = NULL; joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; @@ -955,6 +959,8 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->tuples = 0; joinrel->allvisfrac = 0; joinrel->eclass_indexes = NULL; + joinrel->eclass_source_indexes = NULL; + joinrel->eclass_derive_indexes = NULL; joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 475a65475e..e494309da8 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1194,12 +1194,6 @@ typedef struct RangeTblEntry bool inh; /* inheritance requested? */ bool inFromCl; /* present in FROM clause? */ List *securityQuals; /* security barrier quals to apply, if any */ - Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources - * list for RestrictInfos that mention - * this relation */ - Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives - * list for RestrictInfos that mention - * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 7c94ece332..5571585355 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -943,6 +943,19 @@ typedef struct RelOptInfo double allvisfrac; /* indexes in PlannerInfo's eq_classes list of ECs that mention this rel */ Bitmapset *eclass_indexes; + + /* + * Indexes in PlannerInfo's eq_sources list for RestrictInfos that mention + * this relation. + */ + Bitmapset *eclass_source_indexes; + + /* + * Indexes in PlannerInfo's eq_derives list for RestrictInfos that mention + * this relation. + */ + Bitmapset *eclass_derive_indexes; + PlannerInfo *subroot; /* if subquery */ List *subplan_params; /* if subquery */ /* wanted number of parallel workers */ diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h index 2fa68e59cd..cba6f0be75 100644 --- a/src/include/nodes/readfuncs.h +++ b/src/include/nodes/readfuncs.h @@ -29,8 +29,6 @@ extern PGDLLIMPORT bool restore_location_fields; extern const char *pg_strtok(int *length); extern char *debackslash(const char *token, int length); extern void *nodeRead(const char *token, int tok_len); -extern void pg_strtok_save_context(const char **pcontext); -extern void pg_strtok_restore_context(const char *context); /* * prototypes for functions in readfuncs.c [application/octet-stream] v22-0001-Speed-up-searches-for-child-EquivalenceMembers.patch (61.3K, ../../CAJ2pMkZGB4Yx1dCYkU_YRJgj2rcC0s+PWknqrszmsRPHGLqCgg@mail.gmail.com/3-v22-0001-Speed-up-searches-for-child-EquivalenceMembers.patch) download | inline diff: From 03b82c4f3964ead680b60cdc53905ae4cd8c3309 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:43:36 +0900 Subject: [PATCH v22 1/4] Speed up searches for child EquivalenceMembers Traditionally, child EquivalenceMembers were in the EquivalenceClass->ec_members. When we wanted to find some child members matching a request, we had to perform a linear search. This search became heavy when tables had many partitions, leading to much planning time. After this commit, child EquivalenceMembers no longer exist in ec_members. Instead, RelOptInfos have them. This change demonstrates a significant performance improvement in planning time. --- contrib/postgres_fdw/postgres_fdw.c | 29 +- src/backend/nodes/outfuncs.c | 2 + src/backend/optimizer/path/equivclass.c | 414 ++++++++++++++++++---- src/backend/optimizer/path/indxpath.c | 40 ++- src/backend/optimizer/path/pathkeys.c | 9 +- src/backend/optimizer/plan/analyzejoins.c | 14 +- src/backend/optimizer/plan/createplan.c | 57 +-- src/backend/optimizer/util/inherit.c | 14 + src/backend/optimizer/util/relnode.c | 88 +++++ src/include/nodes/pathnodes.h | 26 ++ src/include/optimizer/pathnode.h | 18 + src/include/optimizer/paths.h | 9 +- 12 files changed, 603 insertions(+), 117 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index e9144beb62..3c3b2cdb34 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7773,13 +7773,27 @@ conversion_error_callback(void *arg) EquivalenceMember * find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) { - ListCell *lc; - PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + Relids top_parent_rel_relids; + List *members; + bool modified; + int i; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + + members = ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + EquivalenceMember *em = list_nth_node(EquivalenceMember, members, i); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_rel_relids)) + add_child_rel_equivalences_to_list(root, ec, em, + rel->relids, + &members, &modified); /* * Note we require !bms_is_empty, else we'd accept constant @@ -7791,6 +7805,8 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) is_foreign_expr(root, rel, em->em_expr)) return em; } + if (unlikely(modified)) + list_free(members); return NULL; } @@ -7844,9 +7860,8 @@ find_em_for_rel_target(PlannerInfo *root, EquivalenceClass *ec, if (em->em_is_const) continue; - /* Ignore child members */ - if (em->em_is_child) - continue; + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* Match if same expression (after stripping relabel) */ em_expr = em->em_expr; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index e66a99247e..8318c71035 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -463,6 +463,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_opfamilies); WRITE_OID_FIELD(ec_collation); WRITE_NODE_FIELD(ec_members); + WRITE_NODE_FIELD(ec_norel_members); + WRITE_NODE_FIELD(ec_rel_members); WRITE_NODE_FIELD(ec_sources); WRITE_NODE_FIELD(ec_derives); WRITE_BITMAPSET_FIELD(ec_relids); diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 7fa502d6e2..7873548b25 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -33,10 +33,14 @@ #include "utils/lsyscache.h" +static EquivalenceMember *make_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + EquivalenceMember *parent, + Oid datatype); static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, - EquivalenceMember *parent, Oid datatype); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, @@ -69,6 +73,12 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo); static JoinDomain *find_join_domain(PlannerInfo *root, Relids relids); +static void add_transformed_child_version(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel, + List **list, + bool *modified); static Bitmapset *get_eclass_indexes_for_relids(PlannerInfo *root, Relids relids); static Bitmapset *get_common_eclass_indexes(PlannerInfo *root, Relids relids1, @@ -342,6 +352,10 @@ process_equivalence(PlannerInfo *root, * be found. */ ec1->ec_members = list_concat(ec1->ec_members, ec2->ec_members); + ec1->ec_norel_members = list_concat(ec1->ec_norel_members, + ec2->ec_norel_members); + ec1->ec_rel_members = list_concat(ec1->ec_rel_members, + ec2->ec_rel_members); ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); @@ -355,6 +369,8 @@ process_equivalence(PlannerInfo *root, root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; + ec2->ec_norel_members = NIL; + ec2->ec_rel_members = NIL; ec2->ec_sources = NIL; ec2->ec_derives = NIL; ec2->ec_relids = NULL; @@ -374,7 +390,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); @@ -391,7 +407,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); @@ -412,6 +428,8 @@ process_equivalence(PlannerInfo *root, ec->ec_opfamilies = opfamilies; ec->ec_collation = collation; ec->ec_members = NIL; + ec->ec_norel_members = NIL; + ec->ec_rel_members = NIL; ec->ec_sources = list_make1(restrictinfo); ec->ec_derives = NIL; ec->ec_relids = NULL; @@ -423,9 +441,9 @@ process_equivalence(PlannerInfo *root, ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; em1 = add_eq_member(ec, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); em2 = add_eq_member(ec, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -511,11 +529,14 @@ canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation) } /* - * add_eq_member - build a new EquivalenceMember and add it to an EC + * make_eq_member + * + * Build a new EquivalenceMember without adding it to an EC. Note that child + * EquivalenceMembers should not be added to its parent EquivalenceClass. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) +make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) { EquivalenceMember *em = makeNode(EquivalenceMember); @@ -526,6 +547,8 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, em->em_datatype = datatype; em->em_jdomain = jdomain; em->em_parent = parent; + em->em_child_relids = NULL; + em->em_child_joinrel_relids = NULL; if (bms_is_empty(relids)) { @@ -546,8 +569,34 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, { ec->ec_relids = bms_add_members(ec->ec_relids, relids); } + + return em; +} + +/* + * add_eq_member - build a new EquivalenceMember and add it to an EC + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + ec->ec_members = lappend(ec->ec_members, em); + /* + * The exact set of relids in the expr for non-child EquivalenceMembers + * as what is given to us in 'relids' should be the same as the relids + * mentioned in the expression. See add_child_rel_equivalences. + */ + /* XXX We need PlannerInfo to use the following assertion */ + /* Assert(bms_equal(pull_varnos(root, (Node *) expr), relids)); */ + if (bms_is_empty(relids)) + ec->ec_norel_members = lappend(ec->ec_norel_members, em); + else + ec->ec_rel_members = lappend(ec->ec_rel_members, em); + return em; } @@ -599,6 +648,17 @@ get_eclass_for_sort_expr(PlannerInfo *root, EquivalenceMember *newem; ListCell *lc1; MemoryContext oldcontext; + Relids top_parent_rel; + + /* + * First, we translate the given Relids to their top-level parents. This is + * required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get child + * members. We can skip such translations if we now see top-level ones, + * i.e., when top_parent_rel is NULL. See the find_relids_top_parents()'s + * definition for more details. + */ + top_parent_rel = find_relids_top_parents(root, rel); /* * Ensure the expression exposes the correct type and collation. @@ -617,7 +677,9 @@ get_eclass_for_sort_expr(PlannerInfo *root, foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); - ListCell *lc2; + List *members; + bool modified; + int i; /* * Never match to a volatile EC, except when we are looking at another @@ -632,16 +694,35 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (!equal(opfamilies, cur_ec->ec_opfamilies)) continue; - foreach(lc2, cur_ec->ec_members) + /* + * When we have to see child EquivalenceMembers, we get and add them to + * 'members'. We cannot use foreach() because the 'members' may be + * modified during iteration. + */ + members = cur_ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); + EquivalenceMember *cur_em = list_nth_node(EquivalenceMember, members, i); + + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them. + */ + if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel)) + add_child_rel_equivalences_to_list(root, cur_ec, cur_em, rel, + &members, &modified); /* * Ignore child members unless they match the request. */ - if (cur_em->em_is_child && - !bms_equal(cur_em->em_relids, rel)) - continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(!cur_em->em_is_child || bms_equal(cur_em->em_relids, rel)); /* * Match constants only within the same JoinDomain (see @@ -654,6 +735,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, equal(expr, cur_em->em_expr)) return cur_ec; /* Match! */ } + if (unlikely(modified)) + list_free(members); } /* No match; does caller want a NULL result? */ @@ -671,6 +754,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = collation; newec->ec_members = NIL; + newec->ec_norel_members = NIL; + newec->ec_rel_members = NIL; newec->ec_sources = NIL; newec->ec_derives = NIL; newec->ec_relids = NULL; @@ -691,7 +776,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, expr_relids = pull_varnos(root, (Node *) expr); newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, NULL, opcintype); + jdomain, opcintype); /* * add_eq_member doesn't check for volatile functions, set-returning @@ -757,19 +842,28 @@ get_eclass_for_sort_expr(PlannerInfo *root, * Child EC members are ignored unless they belong to given 'relids'. */ EquivalenceMember * -find_ec_member_matching_expr(EquivalenceClass *ec, +find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, Expr *expr, Relids relids) { - ListCell *lc; + Relids top_parent_relids; + List *members; + bool modified; + int i; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); /* We ignore binary-compatible relabeling on both ends */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + EquivalenceMember *em = list_nth_node(EquivalenceMember, members, i); Expr *emexpr; /* @@ -779,6 +873,12 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + add_child_rel_equivalences_to_list(root, ec, em, relids, + &members, &modified); + /* * Ignore child members unless they belong to the requested rel. */ @@ -796,6 +896,8 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (equal(emexpr, expr)) return em; } + if (unlikely(modified)) + list_free(members); return NULL; } @@ -828,11 +930,20 @@ find_computable_ec_member(PlannerInfo *root, Relids relids, bool require_parallel_safe) { - ListCell *lc; + Relids top_parent_relids; + List *members; + bool modified; + int i; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + EquivalenceMember *em = list_nth_node(EquivalenceMember, members, i); List *exprvars; ListCell *lc2; @@ -843,6 +954,12 @@ find_computable_ec_member(PlannerInfo *root, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + add_child_rel_equivalences_to_list(root, ec, em, relids, + &members, &modified); + /* * Ignore child members unless they belong to the requested rel. */ @@ -876,6 +993,8 @@ find_computable_ec_member(PlannerInfo *root, return em; /* found usable expression */ } + if (unlikely(modified)) + list_free(members); return NULL; } @@ -939,7 +1058,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, { Expr *targetexpr = (Expr *) lfirst(lc); - em = find_ec_member_matching_expr(ec, targetexpr, rel->relids); + em = find_ec_member_matching_expr(root, ec, targetexpr, rel->relids); if (!em) continue; @@ -1138,7 +1257,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, * machinery might be able to exclude relations on the basis of generated * "var = const" equalities, but "var = param" won't work for that. */ - foreach(lc, ec->ec_members) + foreach(lc, ec->ec_norel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); @@ -1222,7 +1341,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, prev_ems = (EquivalenceMember **) palloc0(root->simple_rel_array_size * sizeof(EquivalenceMember *)); - foreach(lc, ec->ec_members) + foreach(lc, ec->ec_rel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); int relid; @@ -1559,7 +1678,13 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *new_members = NIL; List *outer_members = NIL; List *inner_members = NIL; - ListCell *lc1; + Relids top_parent_join_relids; + List *members; + bool modified; + int i; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_join_relids = find_relids_top_parents(root, join_relids); /* * First, scan the EC to identify member values that are computable at the @@ -1570,9 +1695,19 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * as well as to at least one input member, plus enforce at least one * outer-rel member equal to at least one inner-rel member. */ - foreach(lc1, ec->ec_members) + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = ec->ec_rel_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + EquivalenceMember *cur_em = list_nth_node(EquivalenceMember, members, i); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + bms_is_subset(cur_em->em_relids, top_parent_join_relids)) + add_child_rel_equivalences_to_list(root, ec, cur_em, join_relids, + &members, &modified); /* * We don't need to check explicitly for child EC members. This test @@ -1589,6 +1724,8 @@ generate_join_implied_equalities_normal(PlannerInfo *root, else new_members = lappend(new_members, cur_em); } + if (unlikely(modified)) + list_free(members); /* * First, select the joinclause if needed. We can equate any one outer @@ -1606,6 +1743,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, Oid best_eq_op = InvalidOid; int best_score = -1; RestrictInfo *rinfo; + ListCell *lc1; foreach(lc1, outer_members) { @@ -1680,6 +1818,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *old_members = list_concat(outer_members, inner_members); EquivalenceMember *prev_em = NULL; RestrictInfo *rinfo; + ListCell *lc1; /* For now, arbitrarily take the first old_member as the one to use */ if (old_members) @@ -2177,7 +2316,7 @@ reconsider_outer_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo, * constant before we can decide to throw away the outer-join clause. */ match = false; - foreach(lc2, cur_ec->ec_members) + foreach(lc2, cur_ec->ec_norel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); Oid eq_op; @@ -2285,7 +2424,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) * the COALESCE arguments? */ match = false; - foreach(lc2, cur_ec->ec_members) + foreach(lc2, cur_ec->ec_rel_members) { coal_em = (EquivalenceMember *) lfirst(lc2); Assert(!coal_em->em_is_child); /* no children yet */ @@ -2330,7 +2469,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) * decide to throw away the outer-join clause. */ matchleft = matchright = false; - foreach(lc2, cur_ec->ec_members) + foreach(lc2, cur_ec->ec_norel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); Oid eq_op; @@ -2385,6 +2524,10 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) if (matchleft && matchright) { cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); + Assert(!bms_is_empty(coal_em->em_relids)); + /* XXX performance of list_delete_ptr()?? */ + cur_ec->ec_rel_members = list_delete_ptr(cur_ec->ec_rel_members, + coal_em); return true; } @@ -2455,8 +2598,8 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); if (equal(item1, em->em_expr)) item1member = true; else if (equal(item2, em->em_expr)) @@ -2523,13 +2666,13 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, continue; /* Note: it seems okay to match to "broken" eclasses here */ - foreach(lc2, ec->ec_members) + foreach(lc2, ec->ec_rel_members) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); Var *var; - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* EM must be a Var, possibly with RelabelType */ var = (Var *) em->em_expr; @@ -2626,6 +2769,7 @@ add_child_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_rel->top_parent_relids; Relids child_relids = child_rel->relids; int i; + ListCell *lc; /* * EC merging should be complete already, so we can use the parent rel's @@ -2638,7 +2782,6 @@ add_child_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2651,15 +2794,9 @@ add_child_rel_equivalences(PlannerInfo *root, /* Sanity check eclass_indexes only contain ECs for parent_rel */ Assert(bms_is_subset(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_rel_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2672,8 +2809,8 @@ add_child_rel_equivalences(PlannerInfo *root, * combinations of children. (But add_child_join_rel_equivalences * may add targeted combinations for partitionwise-join purposes.) */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * Consider only members that reference and can be computed at @@ -2689,6 +2826,7 @@ add_child_rel_equivalences(PlannerInfo *root, /* OK, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_rel->reloptkind == RELOPT_BASEREL) { @@ -2718,9 +2856,20 @@ add_child_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, + child_em); + + /* + * We save the knowledge that 'child_em' can be translated from + * 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + cur_em->em_child_relids = bms_add_member(cur_em->em_child_relids, + child_rel->relid); /* Record this EC index for the child rel */ child_rel->eclass_indexes = bms_add_member(child_rel->eclass_indexes, i); @@ -2749,6 +2898,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; MemoryContext oldcontext; + ListCell *lc; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2770,7 +2920,6 @@ add_child_join_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(matching_ecs, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2783,15 +2932,9 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Sanity check on get_eclass_indexes_for_relids result */ Assert(bms_overlap(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_rel_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2800,8 +2943,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, * We consider only original EC members here, not * already-transformed child members. */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * We may ignore expressions that reference a single baserel, @@ -2816,6 +2959,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Yes, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_joinrel->reloptkind == RELOPT_JOINREL) { @@ -2846,9 +2990,21 @@ add_child_join_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_joinrel->eclass_child_members = + lappend(child_joinrel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated from + * 'child_joinrel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + cur_em->em_child_joinrel_relids = + bms_add_member(cur_em->em_child_joinrel_relids, + child_joinrel->join_rel_list_index); } } } @@ -2856,6 +3012,106 @@ add_child_join_rel_equivalences(PlannerInfo *root, MemoryContextSwitchTo(oldcontext); } +/* + * add_transformed_child_version + * Add a transformed EquivalenceMember referencing the given child_rel to + * the list. + * + * This function is expected to be called only from + * add_child_rel_equivalences_to_list(). + */ +static void +add_transformed_child_version(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel, + List **list, + bool *modified) +{ + ListCell *lc; + + foreach(lc, child_rel->eclass_child_members) + { + EquivalenceMember *child_em = lfirst_node(EquivalenceMember, lc); + + /* Skip unwanted EquivalenceMembers */ + if (child_em->em_parent != parent_em) + continue; + + /* + * If this is the first time the given list has been modified, we need to + * make a copy of it. + */ + if (!*modified) + { + *list = list_copy(*list); + *modified = true; + } + + /* Add this child EquivalenceMember to the list */ + *list = lappend(*list, child_em); + } +} + +/* + * add_child_rel_equivalences_to_list + * Add transformed EquivalenceMembers referencing child rels in the given + * child_relids to the list. + * + * The transformation is done in add_transformed_child_version(). + * + * This function will not change the original *list but will always make a copy + * of it when we need to add transformed members. *modified will be true if the + * *list is replaced with a newly allocated one. In such a case, the caller can + * (or must) free the *list. + */ +void +add_child_rel_equivalences_to_list(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids, + List **list, + bool *modified) +{ + int i; + Relids matching_relids; + + /* The given EquivalenceMember should be parent */ + Assert(!parent_em->em_is_child); + + /* + * First, we translate simple rels. + */ + matching_relids = bms_intersect(parent_em->em_child_relids, + child_relids); + i = -1; + while ((i = bms_next_member(matching_relids, i)) >= 0) + { + /* Add transformed child version for this child rel */ + RelOptInfo *child_rel = root->simple_rel_array[i]; + + Assert(child_rel != NULL); + add_transformed_child_version(root, ec, parent_em, child_rel, + list, modified); + } + bms_free(matching_relids); + + /* + * Next, we try to translate join rels. + */ + i = -1; + while ((i = bms_next_member(parent_em->em_child_joinrel_relids, i)) >= 0) + { + /* Add transformed child version for this child join rel */ + RelOptInfo *child_joinrel = + list_nth_node(RelOptInfo, root->join_rel_list, i); + + Assert(child_joinrel != NULL); + add_transformed_child_version(root, ec, parent_em, child_joinrel, + list, modified); + } +} + /* * generate_implied_equalities_for_column @@ -2889,7 +3145,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, { List *result = NIL; bool is_child_rel = (rel->reloptkind == RELOPT_OTHER_MEMBER_REL); - Relids parent_relids; + Relids parent_relids, top_parent_rel_relids; int i; /* Should be OK to rely on eclass_indexes */ @@ -2898,6 +3154,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, /* Indexes are available only on base or "other" member relations. */ Assert(IS_SIMPLE_REL(rel)); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + /* If it's a child rel, we'll need to know what its parent(s) are */ if (is_child_rel) parent_relids = find_childrel_parents(root, rel); @@ -2910,6 +3169,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); EquivalenceMember *cur_em; ListCell *lc2; + List *members; + bool modified; + int j; /* Sanity check eclass_indexes only contain ECs for rel */ Assert(is_child_rel || bms_is_subset(rel->relids, cur_ec->ec_relids)); @@ -2931,15 +3193,25 @@ generate_implied_equalities_for_column(PlannerInfo *root, * corner cases, so for now we live with just reporting the first * match. See also get_eclass_for_sort_expr.) */ + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = cur_ec->ec_rel_members; + modified = false; cur_em = NULL; - foreach(lc2, cur_ec->ec_members) + for (j = 0; j < list_length(members); j++) { - cur_em = (EquivalenceMember *) lfirst(lc2); + cur_em = list_nth_node(EquivalenceMember, members, j); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel_relids)) + add_child_rel_equivalences_to_list(root, cur_ec, cur_em, rel->relids, + &members, &modified); if (bms_equal(cur_em->em_relids, rel->relids) && callback(root, rel, cur_ec, cur_em, callback_arg)) break; cur_em = NULL; } + if (unlikely(modified)) + list_free(members); if (!cur_em) continue; @@ -2954,8 +3226,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, Oid eq_op; RestrictInfo *rinfo; - if (other_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!other_em->em_is_child); /* Make sure it'll be a join to a different rel */ if (other_em == cur_em || @@ -3173,8 +3445,8 @@ eclass_useful_for_merging(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); if (!bms_overlap(cur_em->em_relids, relids)) return true; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 03a5fbdc6d..9fa99e3292 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -184,7 +184,7 @@ static IndexClause *expand_indexqual_rowcompare(PlannerInfo *root, IndexOptInfo *index, Oid expr_op, bool var_on_left); -static void match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +static void match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p); static Expr *match_clause_to_ordering_op(IndexOptInfo *index, @@ -980,7 +980,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * query_pathkeys will allow an incremental sort to be considered on * the index's partially sorted results. */ - match_pathkeys_to_index(index, root->query_pathkeys, + match_pathkeys_to_index(root, index, root->query_pathkeys, &orderbyclauses, &orderbyclausecols); if (list_length(root->query_pathkeys) == list_length(orderbyclauses)) @@ -3071,12 +3071,16 @@ expand_indexqual_rowcompare(PlannerInfo *root, * item in the given 'pathkeys' list. */ static void -match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p) { + Relids top_parent_index_relids; ListCell *lc1; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_index_relids = find_relids_top_parents(root, index->rel->relids); + *orderby_clauses_p = NIL; /* set default results */ *clause_columns_p = NIL; @@ -3088,7 +3092,9 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, { PathKey *pathkey = (PathKey *) lfirst(lc1); bool found = false; - ListCell *lc2; + List *members; + bool modified; + int i; /* Pathkey must request default sort order for the target opfamily */ @@ -3108,15 +3114,33 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, * be considered to match more than one pathkey list, which is OK * here. See also get_eclass_for_sort_expr.) */ - foreach(lc2, pathkey->pk_eclass->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = pathkey->pk_eclass->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *member = (EquivalenceMember *) lfirst(lc2); + EquivalenceMember *member = list_nth_node(EquivalenceMember, members, i); int indexcol; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + bms_equal(member->em_relids, top_parent_index_relids)) + add_child_rel_equivalences_to_list(root, pathkey->pk_eclass, member, + index->rel->relids, + &members, &modified); + /* No possibility of match if it references other relations */ - if (!bms_equal(member->em_relids, index->rel->relids)) + if (member->em_is_child || + !bms_equal(member->em_relids, index->rel->relids)) continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(bms_equal(member->em_relids, index->rel->relids)); + /* * We allow any column of the index to match each pathkey; they * don't have to match left-to-right as you might expect. This is @@ -3145,6 +3169,8 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, if (found) /* don't want to look at remaining members */ break; } + if (unlikely(modified)) + list_free(members); /* * Return the matches found so far when this pathkey couldn't be diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index fdb60aaa8d..cd1be06b5c 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -952,8 +952,8 @@ convert_subquery_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid sub_expr_coll = sub_eclass->ec_collation; ListCell *k; - if (sub_member->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!sub_member->em_is_child); foreach(k, subquery_tlist) { @@ -1479,8 +1479,11 @@ select_outer_pathkeys_for_merge(PlannerInfo *root, { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); + /* Potential future join partner? */ - if (!em->em_is_const && !em->em_is_child && + if (!em->em_is_const && !bms_overlap(em->em_relids, joinrel->relids)) score++; } diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index b9be19a687..bb4ad60084 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -61,7 +61,7 @@ static void remove_leftjoinrel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid); -static void remove_rel_from_eclass(EquivalenceClass *ec, +static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static bool rel_supports_distinctness(PlannerInfo *root, RelOptInfo *rel); @@ -578,7 +578,7 @@ remove_leftjoinrel_from_query(PlannerInfo *root, int relid, if (bms_is_member(relid, ec->ec_relids) || bms_is_member(ojrelid, ec->ec_relids)) - remove_rel_from_eclass(ec, relid, ojrelid); + remove_rel_from_eclass(root, ec, relid, ojrelid); } /* @@ -663,7 +663,8 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) * level(s). */ static void -remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) +remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, + int ojrelid) { ListCell *lc; @@ -687,7 +688,14 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) cur_em->em_relids = bms_del_member(cur_em->em_relids, relid); cur_em->em_relids = bms_del_member(cur_em->em_relids, ojrelid); if (bms_is_empty(cur_em->em_relids)) + { ec->ec_members = foreach_delete_current(ec->ec_members, lc); + /* XXX performance of list_delete_ptr()?? */ + ec->ec_norel_members = list_delete_ptr(ec->ec_norel_members, + cur_em); + ec->ec_rel_members = list_delete_ptr(ec->ec_rel_members, + cur_em); + } } } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 34ca6d4ac2..d67d549b1c 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -260,7 +260,9 @@ static IncrementalSort *make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); -static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -269,9 +271,11 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); -static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Sort *make_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids); -static IncrementalSort *make_incrementalsort_from_pathkeys(Plan *lefttree, +static IncrementalSort *make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, int nPresortedCols); static Sort *make_sort_from_groupcols(List *groupcls, AttrNumber *grpColIdx, @@ -293,7 +297,7 @@ static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); static Unique *make_unique_from_sortclauses(Plan *lefttree, List *distinctList); -static Unique *make_unique_from_pathkeys(Plan *lefttree, +static Unique *make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, int numCols); static Gather *make_gather(List *qptlist, List *qpqual, int nworkers, int rescan_param, bool single_copy, Plan *subplan); @@ -1280,7 +1284,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * function result; it must be the same plan node. However, we then * need to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys((Plan *) plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, (Plan *) plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1324,7 +1328,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * don't need an explicit sort, to make sure they are returning * the same sort key columns the Append expects. */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, nodeSortColIdx, false, @@ -1465,7 +1469,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * function result; it must be the same plan node. However, we then need * to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys(plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1496,7 +1500,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, subplan = create_plan_recurse(root, subpath, CP_EXACT_TLIST); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, node->sortColIdx, false, @@ -1975,7 +1979,7 @@ create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) Assert(pathkeys != NIL); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, best_path->subpath->parent->relids, gm_plan->sortColIdx, false, @@ -2194,7 +2198,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) * relids. Thus, if this sort path is based on a child relation, we must * pass its relids. */ - plan = make_sort_from_pathkeys(subplan, best_path->path.pathkeys, + plan = make_sort_from_pathkeys(root, subplan, best_path->path.pathkeys, IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -2218,7 +2222,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, /* See comments in create_sort_plan() above */ subplan = create_plan_recurse(root, best_path->spath.subpath, flags | CP_SMALL_TLIST); - plan = make_incrementalsort_from_pathkeys(subplan, + plan = make_incrementalsort_from_pathkeys(root, subplan, best_path->spath.path.pathkeys, IS_OTHER_REL(best_path->spath.subpath->parent) ? best_path->spath.path.parent->relids : NULL, @@ -2287,7 +2291,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag subplan = create_plan_recurse(root, best_path->subpath, flags | CP_LABEL_TLIST); - plan = make_unique_from_pathkeys(subplan, + plan = make_unique_from_pathkeys(root, subplan, best_path->path.pathkeys, best_path->numkeys); @@ -4498,7 +4502,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->outersortkeys) { Relids outer_relids = outer_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(outer_plan, + Sort *sort = make_sort_from_pathkeys(root, outer_plan, best_path->outersortkeys, outer_relids); @@ -4512,7 +4516,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->innersortkeys) { Relids inner_relids = inner_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(inner_plan, + Sort *sort = make_sort_from_pathkeys(root, inner_plan, best_path->innersortkeys, inner_relids); @@ -6133,7 +6137,7 @@ make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, * or a Result stacked atop lefttree). */ static Plan * -prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -6200,7 +6204,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ @@ -6228,7 +6232,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr already in tlist */ @@ -6244,7 +6248,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, /* * No matching tlist item; look for a computable expression. */ - em = find_computable_ec_member(NULL, ec, tlist, relids, false); + em = find_computable_ec_member(root, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; @@ -6315,7 +6319,8 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ static Sort * -make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) +make_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + Relids relids) { int numsortkeys; AttrNumber *sortColIdx; @@ -6324,7 +6329,7 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6350,8 +6355,9 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) * 'nPresortedCols' is the number of presorted columns in input tuples */ static IncrementalSort * -make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, - Relids relids, int nPresortedCols) +make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, + List *pathkeys, Relids relids, + int nPresortedCols) { int numsortkeys; AttrNumber *sortColIdx; @@ -6360,7 +6366,7 @@ make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6717,7 +6723,8 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) * as above, but use pathkeys to identify the sort columns and semantics */ static Unique * -make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) +make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + int numCols) { Unique *node = makeNode(Unique); Plan *plan = &node->plan; @@ -6780,7 +6787,7 @@ make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) foreach(j, plan->targetlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, NULL); + em = find_ec_member_matching_expr(root, ec, tle->expr, NULL); if (em) { /* found expr already in tlist */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index f9d3ff1e7a..d826f072c7 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -461,6 +461,7 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, RelationGetRelid(parentrel); Oid childOID = RelationGetRelid(childrel); RangeTblEntry *childrte; + Index topParentRTindex; Index childRTindex; AppendRelInfo *appinfo; TupleDesc child_tupdesc; @@ -568,6 +569,19 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, Assert(root->append_rel_array[childRTindex] == NULL); root->append_rel_array[childRTindex] = appinfo; + /* + * Find a top parent rel's index and save it to top_parent_relid_array. + */ + if (root->top_parent_relid_array == NULL) + root->top_parent_relid_array = + (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); + Assert(root->top_parent_relid_array[childRTindex] == 0); + topParentRTindex = parentRTindex; + while (root->append_rel_array[topParentRTindex] != NULL && + root->append_rel_array[topParentRTindex]->parent_relid != 0) + topParentRTindex = root->append_rel_array[topParentRTindex]->parent_relid; + root->top_parent_relid_array[childRTindex] = topParentRTindex; + /* * Build a PlanRowMark if parent is marked FOR UPDATE/SHARE. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 5d83f60eb9..698ddfd67c 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -122,11 +122,14 @@ setup_simple_rel_arrays(PlannerInfo *root) if (root->append_rel_list == NIL) { root->append_rel_array = NULL; + root->top_parent_relid_array = NULL; return; } root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); + root->top_parent_relid_array = (Index *) + palloc0(size * sizeof(Index)); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -147,6 +150,27 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array[child_relid] = appinfo; } + + /* + * Find a top parent rel's relid for each AppendRelInfo. We cannot do this + * in the last foreach loop because there may be multi-level parent-child + * relations. + */ + for (int i = 0; i < size; i++) + { + int top_parent_relid; + + if (root->append_rel_array[i] == NULL) + continue; + + top_parent_relid = root->append_rel_array[i]->parent_relid; + + while (root->append_rel_array[top_parent_relid] != NULL && + root->append_rel_array[top_parent_relid]->parent_relid != 0) + top_parent_relid = root->append_rel_array[top_parent_relid]->parent_relid; + + root->top_parent_relid_array[i] = top_parent_relid; + } } /* @@ -174,11 +198,23 @@ expand_planner_arrays(PlannerInfo *root, int add_size) repalloc0_array(root->simple_rte_array, RangeTblEntry *, root->simple_rel_array_size, new_size); if (root->append_rel_array) + { root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); + root->top_parent_relid_array = + repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + } else + { root->append_rel_array = palloc0_array(AppendRelInfo *, new_size); + /* + * We do not allocate top_parent_relid_array here so that + * find_relids_top_parents() can early find all of the given Relids are + * top-level. + */ + root->top_parent_relid_array = NULL; + } root->simple_rel_array_size = new_size; } @@ -232,6 +268,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; + rel->eclass_child_members = NIL; rel->serverid = InvalidOid; if (rte->rtekind == RTE_RELATION) { @@ -607,6 +644,12 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) /* GEQO requires us to append the new joinrel to the end of the list! */ root->join_rel_list = lappend(root->join_rel_list, joinrel); + /* + * Store the index of this joinrel to use in + * add_child_join_rel_equivalences(). + */ + joinrel->join_rel_list_index = list_length(root->join_rel_list) - 1; + /* store it into the auxiliary hashtable if there is one. */ if (root->join_rel_hash) { @@ -718,6 +761,7 @@ build_join_rel(PlannerInfo *root, joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -914,6 +958,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -1475,6 +1520,7 @@ fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids) upperrel->cheapest_total_path = NULL; upperrel->cheapest_unique_path = NULL; upperrel->cheapest_parameterized_paths = NIL; + upperrel->eclass_child_members = NIL; /* XXX Is this required? */ root->upper_rels[kind] = lappend(root->upper_rels[kind], upperrel); @@ -1515,6 +1561,48 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } +/* + * find_relids_top_parents_slow + * Slow path of find_relids_top_parents() macro. + * + * Do not call this directly; use the macro instead. See the macro comment for + * more details. + */ +Relids +find_relids_top_parents_slow(PlannerInfo *root, Relids relids) +{ + Index *top_parent_relid_array = root->top_parent_relid_array; + Relids result; + bool is_top_parent; + int i; + + /* This function should be called in the slow path */ + Assert(top_parent_relid_array != NULL); + + result = NULL; + is_top_parent = true; + i = -1; + while ((i = bms_next_member(relids, i)) >= 0) + { + int top_parent_relid = (int) top_parent_relid_array[i]; + + if (top_parent_relid == 0) + top_parent_relid = i; /* 'i' has no parents, so add itself */ + else if (top_parent_relid != i) + is_top_parent = false; + result = bms_add_member(result, top_parent_relid); + } + + if (is_top_parent) + { + bms_free(result); + return NULL; + } + + return result; +} + + /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index ed85dc7414..bc3e1562fc 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -245,6 +245,14 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * append_rel_array is the same length as simple_rel_array and holds the + * top-level parent indexes of the corresponding rels within + * simple_rel_array. The element can be zero if the rel has no parents, + * i.e., is itself in a top-level. + */ + Index *top_parent_relid_array pg_node_attr(read_write_ignore); + /* * all_baserels is a Relids set of all base relids (but not joins or * "other" rels) in the query. This is computed in deconstruct_jointree. @@ -936,6 +944,17 @@ typedef struct RelOptInfo /* Bitmask of optional features supported by the table AM */ uint32 amflags; + /* + * information about a join rel + */ + /* index in root->join_rel_list of this rel */ + Index join_rel_list_index; + + /* + * EquivalenceMembers referencing this rel + */ + List *eclass_child_members; + /* * Information about foreign tables and foreign joins */ @@ -1368,6 +1387,10 @@ typedef struct EquivalenceClass List *ec_opfamilies; /* btree operator family OIDs */ Oid ec_collation; /* collation, if datatypes are collatable */ List *ec_members; /* list of EquivalenceMembers */ + List *ec_norel_members; /* list of EquivalenceMembers whose + * em_relids is empty */ + List *ec_rel_members; /* list of EquivalenceMembers whose + * em_relids is not empty */ List *ec_sources; /* list of generating RestrictInfos */ List *ec_derives; /* list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except @@ -1422,6 +1445,9 @@ typedef struct EquivalenceMember bool em_is_child; /* derived version for a child relation? */ Oid em_datatype; /* the "nominal type" used by the opfamily */ JoinDomain *em_jdomain; /* join domain containing the source clause */ + Relids em_child_relids; /* all relids of child rels */ + Bitmapset *em_child_joinrel_relids; /* indexes in root->join_rel_list of + join rel children */ /* if em_is_child is true, this links to corresponding EM for top parent */ struct EquivalenceMember *em_parent pg_node_attr(read_write_ignore); } EquivalenceMember; diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 6e557bebc4..cae4570823 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -323,6 +323,24 @@ extern Relids min_join_parameterization(PlannerInfo *root, extern RelOptInfo *fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids); extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); + +/* + * find_relids_top_parents + * Compute the set of top-parent relids of rel. + * + * Replaces all Relids appearing in the given 'relids' as their top-level + * parents. The result will be NULL if and only if all of the given relids are + * top-level. + * + * The motivation for having this feature as a macro rather than a function is + * that Relids are top-level in most cases. We can quickly determine when + * root->top_parent_relid_array is NULL. + */ +#define find_relids_top_parents(root, relids) \ + (likely((root)->top_parent_relid_array == NULL) \ + ? NULL : find_relids_top_parents_slow(root, relids)) +extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); + extern ParamPathInfo *get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, Relids required_outer); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9e7408c7ec..5953a0cc26 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -136,7 +136,8 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Index sortref, Relids rel, bool create_it); -extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec, +extern EquivalenceMember *find_ec_member_matching_expr(PlannerInfo *root, + EquivalenceClass *ec, Expr *expr, Relids relids); extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root, @@ -173,6 +174,12 @@ extern void add_child_join_rel_equivalences(PlannerInfo *root, AppendRelInfo **appinfos, RelOptInfo *parent_joinrel, RelOptInfo *child_joinrel); +extern void add_child_rel_equivalences_to_list(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids, + List **list, + bool *modified); extern List *generate_implied_equalities_for_column(PlannerInfo *root, RelOptInfo *rel, ec_matches_callback_type callback, -- 2.42.0.windows.2 [application/octet-stream] v22-0002-Introduce-indexes-for-RestrictInfo.patch (31.6K, ../../CAJ2pMkZGB4Yx1dCYkU_YRJgj2rcC0s+PWknqrszmsRPHGLqCgg@mail.gmail.com/4-v22-0002-Introduce-indexes-for-RestrictInfo.patch) download | inline diff: From 677c951139abe98e851de523ba28d38a16a02b29 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:44:25 +0900 Subject: [PATCH v22 2/4] Introduce indexes for RestrictInfo This change was picked up from v19. Author: David Rowley <[email protected]> and me --- src/backend/nodes/outfuncs.c | 6 +- src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/path/equivclass.c | 338 +++++++++++++++++++--- src/backend/optimizer/plan/analyzejoins.c | 8 +- src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/inherit.c | 7 + src/include/nodes/parsenodes.h | 6 + src/include/nodes/pathnodes.h | 47 ++- src/include/optimizer/paths.h | 15 +- 11 files changed, 383 insertions(+), 53 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 8318c71035..1904a53acb 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -465,8 +465,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_members); WRITE_NODE_FIELD(ec_norel_members); WRITE_NODE_FIELD(ec_rel_members); - WRITE_NODE_FIELD(ec_sources); - WRITE_NODE_FIELD(ec_derives); + WRITE_BITMAPSET_FIELD(ec_source_indexes); + WRITE_BITMAPSET_FIELD(ec_derive_indexes); WRITE_BITMAPSET_FIELD(ec_relids); WRITE_BOOL_FIELD(ec_has_const); WRITE_BOOL_FIELD(ec_has_volatile); @@ -569,6 +569,8 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(inh); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); + WRITE_BITMAPSET_FIELD(eclass_source_indexes); + WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index cc2021c1f7..7610447ad3 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -574,6 +574,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); + READ_BITMAPSET_FIELD(eclass_source_indexes); + READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index d6ceafd51c..fcda83ad2c 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5755,7 +5755,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, if (ec && ec->ec_has_const) { EquivalenceMember *em = fkinfo->fk_eclass_member[i]; - RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + RestrictInfo *rinfo = find_derived_clause_for_ec_member(root, + ec, em); if (rinfo) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 7873548b25..813a365e63 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -42,6 +42,10 @@ static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, Oid datatype); +static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); +static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); @@ -321,7 +325,6 @@ process_equivalence(PlannerInfo *root, /* If case 1, nothing to do, except add to sources */ if (ec1 == ec2) { - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -332,6 +335,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); return true; } @@ -356,8 +361,10 @@ process_equivalence(PlannerInfo *root, ec2->ec_norel_members); ec1->ec_rel_members = list_concat(ec1->ec_rel_members, ec2->ec_rel_members); - ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); - ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); + ec1->ec_source_indexes = bms_join(ec1->ec_source_indexes, + ec2->ec_source_indexes); + ec1->ec_derive_indexes = bms_join(ec1->ec_derive_indexes, + ec2->ec_derive_indexes); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); ec1->ec_has_const |= ec2->ec_has_const; /* can't need to set has_volatile */ @@ -371,10 +378,9 @@ process_equivalence(PlannerInfo *root, ec2->ec_members = NIL; ec2->ec_norel_members = NIL; ec2->ec_rel_members = NIL; - ec2->ec_sources = NIL; - ec2->ec_derives = NIL; + ec2->ec_source_indexes = NULL; + ec2->ec_derive_indexes = NULL; ec2->ec_relids = NULL; - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -385,13 +391,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec1) { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, jdomain, item2_type); - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -402,13 +409,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec2) { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, jdomain, item1_type); - ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -419,6 +427,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec2, restrictinfo); } else { @@ -430,8 +440,8 @@ process_equivalence(PlannerInfo *root, ec->ec_members = NIL; ec->ec_norel_members = NIL; ec->ec_rel_members = NIL; - ec->ec_sources = list_make1(restrictinfo); - ec->ec_derives = NIL; + ec->ec_source_indexes = NULL; + ec->ec_derive_indexes = NULL; ec->ec_relids = NULL; ec->ec_has_const = false; ec->ec_has_volatile = false; @@ -453,6 +463,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec, restrictinfo); } return true; @@ -600,6 +612,50 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, return em; } +/* + * add_eq_source - add 'rinfo' in eq_sources for this 'ec' + */ +static void +add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int source_idx = list_length(root->eq_sources); + int i; + + ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); + root->eq_sources = lappend(root->eq_sources, rinfo); + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + source_idx); + } +} + +/* + * add_eq_derive - add 'rinfo' in eq_derives for this 'ec' + */ +static void +add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int derive_idx = list_length(root->eq_derives); + int i; + + ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); + root->eq_derives = lappend(root->eq_derives, rinfo); + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + derive_idx); + } +} + /* * get_eclass_for_sort_expr @@ -756,8 +812,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_members = NIL; newec->ec_norel_members = NIL; newec->ec_rel_members = NIL; - newec->ec_sources = NIL; - newec->ec_derives = NIL; + newec->ec_source_indexes = NULL; + newec->ec_derive_indexes = NULL; newec->ec_relids = NULL; newec->ec_has_const = false; newec->ec_has_volatile = contain_volatile_functions((Node *) expr); @@ -1145,7 +1201,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources or ec_derives for matches. It doesn't really + * don't search eq_sources or eq_derives for matches. It doesn't really * seem worth the trouble to do so. */ void @@ -1234,6 +1290,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *const_em = NULL; ListCell *lc; + int i; /* * In the trivial case where we just had one "var = const" clause, push @@ -1243,9 +1300,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, * equivalent to the old one. */ if (list_length(ec->ec_members) == 2 && - list_length(ec->ec_sources) == 1) + bms_get_singleton_member(ec->ec_source_indexes, &i)) { - RestrictInfo *restrictinfo = (RestrictInfo *) linitial(ec->ec_sources); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, root->eq_sources, i); distribute_restrictinfo_to_rels(root, restrictinfo); return; @@ -1303,9 +1360,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the correct - * markings for a mergejoinable clause, and save it in ec_derives. (We + * markings for a mergejoinable clause, and save it in eq_derives. (We * will not re-use such clauses directly, but selectivity estimation - * may consult the list later. Note that this use of ec_derives does + * may consult the list later. Note that this use of eq_derives does * not overlap with its use for join clauses, since we never generate * join clauses from an ec_has_const eclass.) */ @@ -1315,7 +1372,8 @@ generate_base_implied_equalities_const(PlannerInfo *root, rinfo->left_ec = rinfo->right_ec = ec; rinfo->left_em = cur_em; rinfo->right_em = const_em; - ec->ec_derives = lappend(ec->ec_derives, rinfo); + + add_eq_derive(root, ec, rinfo); } } } @@ -1381,7 +1439,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the * correct markings for a mergejoinable clause. We don't put it - * in ec_derives however; we don't currently need to re-find such + * in eq_derives however; we don't currently need to re-find such * clauses, and we don't want to clutter that list with non-join * clauses. */ @@ -1437,11 +1495,12 @@ static void generate_base_implied_equalities_broken(PlannerInfo *root, EquivalenceClass *ec) { - ListCell *lc; + int i = -1; - foreach(lc, ec->ec_sources) + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); if (ec->ec_has_const || bms_membership(restrictinfo->required_relids) != BMS_MULTIPLE) @@ -1482,11 +1541,11 @@ generate_base_implied_equalities_broken(PlannerInfo *root, * Because the same join clauses are likely to be needed multiple times as * we consider different join paths, we avoid generating multiple copies: * whenever we select a particular pair of EquivalenceMembers to join, - * we check to see if the pair matches any original clause (in ec_sources) - * or previously-built clause (in ec_derives). This saves memory and allows - * re-use of information cached in RestrictInfos. We also avoid generating - * commutative duplicates, i.e. if the algorithm selects "a.x = b.y" but - * we already have "b.y = a.x", we return the existing clause. + * we check to see if the pair matches any original clause in root's + * eq_sources or previously-built clause (in root's eq_derives). This saves + * memory and allows re-use of information cached in RestrictInfos. We also + * avoid generating commutative duplicates, i.e. if the algorithm selects + * "a.x = b.y" but we already have "b.y = a.x", we return the existing clause. * * If we are considering an outer join, sjinfo is the associated OJ info, * otherwise it can be NULL. @@ -1871,12 +1930,16 @@ generate_join_implied_equalities_broken(PlannerInfo *root, Relids nominal_inner_relids, RelOptInfo *inner_rel) { + Bitmapset *matching_es; List *result = NIL; - ListCell *lc; + int i; - foreach(lc, ec->ec_sources) + matching_es = get_ec_source_indexes(root, ec, nominal_join_relids); + i = -1; + while ((i = bms_next_member(matching_es, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); Relids clause_relids = restrictinfo->required_relids; if (bms_is_subset(clause_relids, nominal_join_relids) && @@ -1888,12 +1951,12 @@ generate_join_implied_equalities_broken(PlannerInfo *root, /* * If we have to translate, just brute-force apply adjust_appendrel_attrs * to all the RestrictInfos at once. This will result in returning - * RestrictInfos that are not listed in ec_derives, but there shouldn't be + * RestrictInfos that are not listed in eq_derives, but there shouldn't be * any duplication, and it's a sufficiently narrow corner case that we * shouldn't sweat too much over it anyway. * * Since inner_rel might be an indirect descendant of the baserel - * mentioned in the ec_sources clauses, we have to be prepared to apply + * mentioned in the eq_sources clauses, we have to be prepared to apply * multiple levels of Var translation. */ if (IS_OTHER_REL(inner_rel) && result != NIL) @@ -1955,10 +2018,11 @@ create_join_clause(PlannerInfo *root, EquivalenceMember *rightem, EquivalenceClass *parent_ec) { + Bitmapset *matches; RestrictInfo *rinfo; RestrictInfo *parent_rinfo = NULL; - ListCell *lc; MemoryContext oldcontext; + int i; /* * Search to see if we already built a RestrictInfo for this pair of @@ -1969,9 +2033,12 @@ create_join_clause(PlannerInfo *root, * it's not identical, it'd better have the same effects, or the operator * families we're using are broken. */ - foreach(lc, ec->ec_sources) + matches = bms_int_members(get_ec_source_indexes_strict(root, ec, leftem->em_relids), + get_ec_source_indexes_strict(root, ec, rightem->em_relids)); + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -1982,9 +2049,13 @@ create_join_clause(PlannerInfo *root, return rinfo; } - foreach(lc, ec->ec_derives) + matches = bms_int_members(get_ec_derive_indexes_strict(root, ec, leftem->em_relids), + get_ec_derive_indexes_strict(root, ec, rightem->em_relids)); + + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -2042,7 +2113,7 @@ create_join_clause(PlannerInfo *root, rinfo->left_em = leftem; rinfo->right_em = rightem; /* and save it for possible re-use */ - ec->ec_derives = lappend(ec->ec_derives, rinfo); + add_eq_derive(root, ec, rinfo); MemoryContextSwitchTo(oldcontext); @@ -2721,16 +2792,19 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, * Returns NULL if no such clause can be found. */ RestrictInfo * -find_derived_clause_for_ec_member(EquivalenceClass *ec, +find_derived_clause_for_ec_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em) { - ListCell *lc; + int i; Assert(ec->ec_has_const); Assert(!em->em_is_const); - foreach(lc, ec->ec_derives) + + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); /* * generate_base_implied_equalities_const will have put non-const @@ -3335,7 +3409,7 @@ have_relevant_eclass_joinclause(PlannerInfo *root, * surely be true if both of them overlap ec_relids.) * * Note we don't test ec_broken; if we did, we'd need a separate code - * path to look through ec_sources. Checking the membership anyway is + * path to look through eq_sources. Checking the membership anyway is * OK as a possibly-overoptimistic heuristic. * * We don't test ec_has_const either, even though a const eclass won't @@ -3423,7 +3497,7 @@ eclass_useful_for_merging(PlannerInfo *root, /* * Note we don't test ec_broken; if we did, we'd need a separate code path - * to look through ec_sources. Checking the members anyway is OK as a + * to look through eq_sources. Checking the members anyway is OK as a * possibly-overoptimistic heuristic. */ @@ -3576,3 +3650,179 @@ get_common_eclass_indexes(PlannerInfo *root, Relids relids1, Relids relids2) /* Calculate and return the common EC indexes, recycling the left input. */ return bms_int_members(rel1ecs, rel2ecs); } + +/* + * get_ec_source_indexes + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_esis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_esis, ec->ec_source_indexes); +} + +/* + * get_ec_source_indexes_strict + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *esis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + esis = bms_intersect(ec->ec_source_indexes, + rte->eclass_source_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + esis = bms_int_members(esis, rte->eclass_source_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return esis; +} + +/* + * get_ec_derive_indexes + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + * + * XXX is this function even needed? + */ +Bitmapset * +get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_edis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_edis, ec->ec_derive_indexes); +} + +/* + * get_ec_derive_indexes_strict + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *edis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + edis = bms_intersect(ec->ec_derive_indexes, + rte->eclass_derive_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + edis = bms_int_members(edis, rte->eclass_derive_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return edis; +} diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index bb4ad60084..4ea0cfd803 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -667,6 +667,7 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid) { ListCell *lc; + int i; /* Fix up the EC's overall relids */ ec->ec_relids = bms_del_member(ec->ec_relids, relid); @@ -700,9 +701,10 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, } /* Fix up the source clauses, in case we can re-use them later */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); remove_rel_from_restrictinfo(rinfo, relid, ojrelid); } @@ -712,7 +714,7 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, * drop them. (At this point, any such clauses would be base restriction * clauses, which we'd not need anymore anyway.) */ - ec->ec_derives = NIL; + ec->ec_derive_indexes = NULL; } /* diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index a8cea5efe1..773d703c88 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -646,6 +646,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, root->multiexpr_params = NIL; root->join_domains = NIL; root->eq_classes = NIL; + root->eq_sources = NIL; + root->eq_derives = NIL; root->ec_merging_done = false; root->last_rinfo_serial = 0; root->all_result_relids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 73ff40721c..1cce43a94e 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -993,6 +993,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, subroot->multiexpr_params = NIL; subroot->join_domains = NIL; subroot->eq_classes = NIL; + subroot->eq_sources = NIL; + subroot->eq_derives = NIL; subroot->ec_merging_done = false; subroot->last_rinfo_serial = 0; subroot->all_result_relids = NULL; diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index d826f072c7..b75e92b2e1 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -483,6 +483,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); + /* + * We do not want to inherit the EquivalenceMember indexes of the parent + * to its child + */ + childrte->eclass_source_indexes = NULL; + childrte->eclass_derive_indexes = NULL; + Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index e494309da8..475a65475e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1194,6 +1194,12 @@ typedef struct RangeTblEntry bool inh; /* inheritance requested? */ bool inFromCl; /* present in FROM clause? */ List *securityQuals; /* security barrier quals to apply, if any */ + Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources + * list for RestrictInfos that mention + * this relation */ + Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives + * list for RestrictInfos that mention + * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index bc3e1562fc..7c94ece332 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -318,6 +318,12 @@ struct PlannerInfo /* list of active EquivalenceClasses */ List *eq_classes; + /* list of source RestrictInfos used to build EquivalenceClasses */ + List *eq_sources; + + /* list of RestrictInfos derived from EquivalenceClasses */ + List *eq_derives; + /* set true once ECs are canonical */ bool ec_merging_done; @@ -1369,6 +1375,41 @@ typedef struct JoinDomain * entry: consider SELECT random() AS a, random() AS b ... ORDER BY b,a. * So we record the SortGroupRef of the originating sort clause. * + * TODO: We should update the following comments. + * + * At various locations in the query planner, we must search for + * EquivalenceMembers within a given EquivalenceClass. For the common case, + * an EquivalenceClass does not have a large number of EquivalenceMembers, + * however, in cases such as planning queries to partitioned tables, the number + * of members can become large. To maintain planning performance, we make use + * of a bitmap index to allow us to quickly find EquivalenceMembers in a given + * EquivalenceClass belonging to a given relation or set of relations. This + * is done by storing a list of EquivalenceMembers belonging to all + * EquivalenceClasses in PlannerInfo and storing a Bitmapset for each + * RelOptInfo which has a bit set for each EquivalenceMember in that list + * which relates to the given relation. We also store a Bitmapset to mark all + * of the indexes in the PlannerInfo's list of EquivalenceMembers in the + * EquivalenceClass. We can quickly find the interesting indexes into the + * PlannerInfo's list by performing a bit-wise AND on the RelOptInfo's + * Bitmapset and the EquivalenceClasses. + * + * To further speed up the lookup of EquivalenceMembers we also record the + * non-child indexes. This allows us to deal with fewer bits for searches + * that don't require "is_child" EquivalenceMembers. We must also store the + * indexes into EquivalenceMembers which have no relids mentioned as some + * searches require that. + * + * Additionally, we also store the EquivalenceMembers of a given + * EquivalenceClass in the ec_members list. Technically, we could obtain this + * from looking at the bits in ec_member_indexes, however, finding all members + * of a given EquivalenceClass is common enough that it pays to have fast + * access to a dense list containing all members. + * + * The source and derived RestrictInfos are indexed in a similar method, + * although we don't maintain a List in the EquivalenceClass for these. These + * must be looked up in PlannerInfo using the ec_source_indexes and + * ec_derive_indexes Bitmapsets. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1391,8 +1432,10 @@ typedef struct EquivalenceClass * em_relids is empty */ List *ec_rel_members; /* list of EquivalenceMembers whose * em_relids is not empty */ - List *ec_sources; /* list of generating RestrictInfos */ - List *ec_derives; /* list of derived RestrictInfos */ + Bitmapset *ec_source_indexes; /* indexes into PlannerInfo's eq_sources + * list of generating RestrictInfos */ + Bitmapset *ec_derive_indexes; /* indexes into PlannerInfo's eq_derives + * list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except * for child members (see below) */ bool ec_has_const; /* any pseudoconstants in ec_members? */ diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 5953a0cc26..97c19d0068 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -163,7 +163,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2); extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); -extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, +extern RestrictInfo *find_derived_clause_for_ec_member(PlannerInfo *root, + EquivalenceClass *ec, EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, @@ -195,6 +196,18 @@ extern bool eclass_useful_for_merging(PlannerInfo *root, extern bool is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist); extern bool is_redundant_with_indexclauses(RestrictInfo *rinfo, List *indexclauses); +extern Bitmapset *get_ec_source_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_source_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); /* * pathkeys.c -- 2.42.0.windows.2 [application/octet-stream] v22-0003-Solve-conflict-with-self-join-removal.patch (3.8K, ../../CAJ2pMkZGB4Yx1dCYkU_YRJgj2rcC0s+PWknqrszmsRPHGLqCgg@mail.gmail.com/5-v22-0003-Solve-conflict-with-self-join-removal.patch) download | inline diff: From bd767c17d3cc4836867c1a2e1dc039b5ce1fad82 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Mon, 11 Dec 2023 12:19:55 +0900 Subject: [PATCH v22 3/4] Solve conflict with self join removal Written by Alena Rybakina <[email protected]> [1] with modifications by me. [1] https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/optimizer/plan/analyzejoins.c | 49 ++++++++++++++++------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 4ea0cfd803..31f0c717a2 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -1561,10 +1561,12 @@ replace_relid(Relids relids, int oldId, int newId) * delete them. */ static void -update_eclasses(EquivalenceClass *ec, int from, int to) +update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) { List *new_members = NIL; - List *new_sources = NIL; + Bitmapset *new_source_indexes = NULL; + int i; + int j; ListCell *lc; ListCell *lc1; @@ -1606,18 +1608,28 @@ update_eclasses(EquivalenceClass *ec, int from, int to) list_free(ec->ec_members); ec->ec_members = new_members; - list_free(ec->ec_derives); - ec->ec_derives = NULL; + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) + { + /* + * Can't delete the element because we would need to rebuild all + * the eq_derives indexes. But set a nuke to detect potential problems. + */ + list_nth_cell(root->eq_derives, i)->ptr_value = NULL; + } + bms_free(ec->ec_derive_indexes); + ec->ec_derive_indexes = NULL; /* Update EC source expressions */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); bool is_redundant = false; if (!bms_is_member(from, rinfo->required_relids)) { - new_sources = lappend(new_sources, rinfo); + new_source_indexes = bms_add_member(new_source_indexes, i); continue; } @@ -1628,9 +1640,10 @@ update_eclasses(EquivalenceClass *ec, int from, int to) * redundancy with existing ones. We don't have to check for * redundancy with derived clauses, because we've just deleted them. */ - foreach(lc1, new_sources) + j = -1; + while ((j = bms_next_member(new_source_indexes, j)) >= 0) { - RestrictInfo *other = lfirst_node(RestrictInfo, lc1); + RestrictInfo *other = list_nth_node(RestrictInfo, root->eq_sources, j); if (!equal(rinfo->clause_relids, other->clause_relids)) continue; @@ -1642,12 +1655,20 @@ update_eclasses(EquivalenceClass *ec, int from, int to) } } - if (!is_redundant) - new_sources = lappend(new_sources, rinfo); + if (is_redundant) + { + /* + * Can't delete the element because we would need to rebuild all + * the eq_sources indexes. But set a nuke to detect potential problems. + */ + list_nth_cell(root->eq_sources, i)->ptr_value = NULL; + } + else + new_source_indexes = bms_add_member(new_source_indexes, i); } - list_free(ec->ec_sources); - ec->ec_sources = new_sources; + bms_free(ec->ec_source_indexes); + ec->ec_source_indexes = new_source_indexes; ec->ec_relids = replace_relid(ec->ec_relids, from, to); } @@ -1825,7 +1846,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, { EquivalenceClass *ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - update_eclasses(ec, toRemove->relid, toKeep->relid); + update_eclasses(root, ec, toRemove->relid, toKeep->relid); toKeep->eclass_indexes = bms_add_member(toKeep->eclass_indexes, i); } -- 2.42.0.windows.2 [application/octet-stream] v22-0004-Fix-a-bug-related-to-atomic-function.patch (3.0K, ../../CAJ2pMkZGB4Yx1dCYkU_YRJgj2rcC0s+PWknqrszmsRPHGLqCgg@mail.gmail.com/6-v22-0004-Fix-a-bug-related-to-atomic-function.patch) download | inline diff: From e6e38cb16c793efdb17f36078a7ff9fe83610953 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Mon, 11 Dec 2023 12:20:20 +0900 Subject: [PATCH v22 4/4] Fix a bug related to atomic function Author: Alena Rybakina <[email protected]> https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/nodes/read.c | 20 ++++++++++++++++++++ src/backend/nodes/readfuncs.c | 24 ++++++++++++++++++++++-- src/include/nodes/readfuncs.h | 2 ++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c index 813eda3e73..314736d989 100644 --- a/src/backend/nodes/read.c +++ b/src/backend/nodes/read.c @@ -514,3 +514,23 @@ nodeRead(const char *token, int tok_len) return (void *) result; } + +/* + * pg_strtok_save_context - + * Save context initialized by stringToNode. + */ +void +pg_strtok_save_context(const char **pcontext) +{ + *pcontext = pg_strtok_ptr; +} + +/* + * pg_strtok_restore_context - + * Resore saved context. + */ +void +pg_strtok_restore_context(const char *context) +{ + pg_strtok_ptr = context; +} diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 7610447ad3..0915c0e661 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -191,6 +191,26 @@ nullable_string(const char *token, int length) return debackslash(token, length); } +/* Read an equivalence field (anything written as ":fldname %u") and check it */ +#define READ_EQ_BITMAPSET_FIELD_CHECK(fldname) \ +{ \ + int save_length = length; \ + const char *context; \ + pg_strtok_save_context(&context); \ + token = pg_strtok(&length); \ + if (length > 0 && strncmp(token, ":"#fldname, strlen(":"#fldname))) \ + { \ + /* "fldname" field was not found - fill it and restore context. */ \ + local_node->fldname = NULL; \ + pg_strtok_restore_context(context); \ + length = save_length; \ + } \ + else \ + { \ + local_node->fldname = _readBitmapset(); \ + } \ +} + /* * _readBitmapset @@ -574,8 +594,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_BITMAPSET_FIELD(eclass_source_indexes); - READ_BITMAPSET_FIELD(eclass_derive_indexes); + READ_EQ_BITMAPSET_FIELD_CHECK(eclass_source_indexes); + READ_EQ_BITMAPSET_FIELD_CHECK(eclass_derive_indexes); READ_DONE(); } diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h index cba6f0be75..2fa68e59cd 100644 --- a/src/include/nodes/readfuncs.h +++ b/src/include/nodes/readfuncs.h @@ -29,6 +29,8 @@ extern PGDLLIMPORT bool restore_location_fields; extern const char *pg_strtok(int *length); extern char *debackslash(const char *token, int length); extern void *nodeRead(const char *token, int tok_len); +extern void pg_strtok_save_context(const char **pcontext); +extern void pg_strtok_restore_context(const char *context); /* * prototypes for functions in readfuncs.c -- 2.42.0.windows.2 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2023-12-16 15:41 Alena Rybakina <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Alena Rybakina @ 2023-12-16 15:41 UTC (permalink / raw) To: Yuya Watari <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; Ashutosh Bapat <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hi! On 13.12.2023 09:21, Yuya Watari wrote: > Hello Alena, Andrei, and all, > > I am sorry for my late response. I found that the current patches do > not apply to the master, so I have rebased those patches. I have > attached v22. For this later discussion, I separated the rebasing and > bug fixing that Alena did in v21 into separate commits, v22-0003 and > v22-0004. I will merge these commits after the discussion. > > 1. v22-0003 (solved_conflict_with_self_join_removal.txt) Thank you! > Thank you for your rebase. Looking at your rebasing patch, I thought > we could do this more simply. Your patch deletes (more precisely, sets > to null) non-redundant members from the root->eq_sources list and > re-adds them to the same list. However, this approach seems a little > waste of memory. Instead, we can update > EquivalenceClass->ec_source_indexes directly. Then, we can reuse the > members in root->eq_sources and don't need to extend root->eq_sources. > I did this in v22-0003. What do you think of this approach? I thought about this earlier and was worried that the index links of the equivalence classes might not be referenced correctly for outer joins, so I decided to just overwrite them and reset the previous ones. > The main concern with this idea is that it does not fix > RangeTblEntry->eclass_source_indexes. The current code works fine even > if we don't fix the index because get_ec_source_indexes() always does > bms_intersect() for eclass_source_indexes and ec_source_indexes. If we > guaranteed this behavior of doing bms_intersect, then simply modifying > ec_source_indexes would be fine. Fortunately, such a guarantee is not > so difficult. > > And your patch removes the following assertion code from the previous > patch. May I ask why you removed this code? I think this assertion is > helpful for sanity checks. Of course, I know that this kind of > assertion will slow down regression tests or assert-enabled builds. > So, we may have to discuss which assertions to keep and which to > discard. > > ===== > -#ifdef USE_ASSERT_CHECKING > - /* verify the results look sane */ > - i = -1; > - while ((i = bms_next_member(rel_esis, i)) >= 0) > - { > - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, > - i); > - > - Assert(bms_overlap(relids, rinfo->clause_relids)); > - } > -#endif > ===== this is due to the fact that I explained before: we zeroed the values indicated by the indexes, then this check is not correct either - since the zeroed value indicated by the index is correct. That's why I removed this check. > Finally, your patch changes the name of the following function. I > understand the need for this change, but it has nothing to do with our > patches, so we should not include it and discuss it in another thread. > > ===== > -update_eclasses(EquivalenceClass *ec, int from, int to) > +update_eclass(PlannerInfo *root, EquivalenceClass *ec, int from, int to) > ===== I agree. > 2. v22-0004 (bug_related_to_atomic_function.txt) > > Thank you for fixing the bug. As I wrote in the previous mail: > > On Wed, Nov 22, 2023 at 2:32 PM Yuya Watari<[email protected]> wrote: >> On Mon, Nov 20, 2023 at 1:45 PM Andrei Lepikhov >> <[email protected]> wrote: >>> During the work on committing the SJE feature [1], Alexander Korotkov >>> pointed out the silver lining in this work [2]: he proposed that we >>> shouldn't remove RelOptInfo from simple_rel_array at all but replace it >>> with an 'Alias', which will refer the kept relation. It can simplify >>> further optimizations on removing redundant parts of the query. >> Thank you for sharing this information. I think the idea suggested by >> Alexander Korotkov is also helpful for our patch. As mentioned above, >> the indexes are in RangeTblEntry in the current implementation. >> However, I think RangeTblEntry is not the best place to store them. An >> 'alias' relids may help solve this and simplify fixing the above bug. >> I will try this approach soon. > I think that the best way to solve this issue is to move these indexes > from RangeTblEntry to RelOptInfo. Since they are related to planning > time, they should be in RelOptInfo. The reason why I put these indexes > in RangeTblEntry is because some RelOptInfos can be null and we cannot > store the indexes. This problem is similar to an issue regarding > 'varno 0' Vars. I hope an alias RelOptInfo would help solve this > issue. I have attached the current proof of concept I am considering > as poc-alias-reloptinfo.txt. To test this patch, please follow the > procedure below. > > 1. Apply all *.patch files, > 2. Apply Alexander Korotkov's alias_relids.patch [1], and > 3. Apply poc-alias-reloptinfo.txt, which is attached to this email. > > My patch creates a dummy (or an alias) RelOptInfo to store indexes if > the corresponding RelOptInfo is null. The following is the core change > in my patch. > > ===== > @@ -627,9 +627,19 @@ add_eq_source(PlannerInfo *root, EquivalenceClass > *ec, RestrictInfo *rinfo) > i = -1; > while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) > { > - RangeTblEntry *rte = root->simple_rte_array[i]; > + RelOptInfo *rel = root->simple_rel_array[i]; > > - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, > + /* > + * If the corresponding RelOptInfo does not exist, we create a 'dummy' > + * RelOptInfo for storing EquivalenceClass indexes. > + */ > + if (rel == NULL) > + { > + rel = root->simple_rel_array[i] = makeNode(RelOptInfo); > + rel->eclass_source_indexes = NULL; > + rel->eclass_derive_indexes = NULL; > + } > + rel->eclass_source_indexes = bms_add_member(rel->eclass_source_indexes, > source_idx); > } > ===== > > At this point, I'm not sure if this approach is correct. It seems to > pass the regression tests, but we should doubt its correctness. I will > continue to experiment with this idea. > > [1]https://www.postgresql.org/message-id/CAPpHfdseB13zJJPZuBORevRnZ0vcFyUaaJeSGfAysX7S5er%2BEQ%40mail.g... > Yes, I also thought in this direction before and I agree that this is the best way to develop the patch. -- Regards, Alena Rybakina Postgres Professional:http://www.postgrespro.com The Russian Postgres Company ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-01-17 09:33 Yuya Watari <[email protected]> parent: Alena Rybakina <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-01-17 09:33 UTC (permalink / raw) To: Alena Rybakina <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; Ashutosh Bapat <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hello Alena, Thank you for your quick response, and I'm sorry for my delayed reply. On Sun, Dec 17, 2023 at 12:41 AM Alena Rybakina <[email protected]> wrote: > I thought about this earlier and was worried that the index links of the equivalence classes might not be referenced correctly for outer joins, > so I decided to just overwrite them and reset the previous ones. Thank you for pointing this out. I have investigated this problem and found a potential bug place. The code quoted below modifies RestrictInfo's clause_relids. Here, our indexes, namely eclass_source_indexes and eclass_derive_indexes, are based on clause_relids, so they should be adjusted after the modification. However, my patch didn't do that, so it may have missed some references. The same problem occurs in places other than the quoted one. ===== /* * Walker function for replace_varno() */ static bool replace_varno_walker(Node *node, ReplaceVarnoContext *ctx) { ... else if (IsA(node, RestrictInfo)) { RestrictInfo *rinfo = (RestrictInfo *) node; ... if (bms_is_member(ctx->from, rinfo->clause_relids)) { replace_varno((Node *) rinfo->clause, ctx->from, ctx->to); replace_varno((Node *) rinfo->orclause, ctx->from, ctx->to); rinfo->clause_relids = replace_relid(rinfo->clause_relids, ctx->from, ctx->to); ... } ... } ... } ===== I have attached a new version of the patch, v23, to fix this problem. v23-0006 adds a helper function called update_clause_relids(). This function modifies RestrictInfo->clause_relids while adjusting its related indexes. I have also attached a sanity check patch (sanity-check.txt) to this email. This sanity check patch verifies that there are no missing references between RestrictInfos and our indexes. I don't intend to commit this patch, but it helps find potential bugs. v23 passes this sanity check, but the v21 you submitted before does not. This means that the adjustment by update_clause_relids() is needed to prevent missing references after modifying clause_relids. I'd appreciate your letting me know if v23 doesn't solve your concern. One of the things I don't think is good about my approach is that it adds some complexity to the code. In my approach, all modifications to clause_relids must be done through the update_clause_relids() function, but enforcing this rule is not so easy. In this sense, my patch may need to be simplified more. > this is due to the fact that I explained before: we zeroed the values indicated by the indexes, > then this check is not correct either - since the zeroed value indicated by the index is correct. > That's why I removed this check. Thank you for letting me know. I fixed this in v23-0005 to adjust the indexes in update_eclasses(). With this change, the assertion check will be correct. -- Best regards, Yuya Watari diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 2da2a6c14e..334ea8d3bb 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -1751,6 +1751,48 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) bms_free(ec->ec_source_indexes); ec->ec_source_indexes = new_source_indexes; ec->ec_relids = replace_relid(ec->ec_relids, from, to); + +#ifdef USE_ASSERT_CHECKING + /* + * Sanity check: + * Verify that there are no missing references between RestrictInfos and + * our indexes, namely eclass_source_indexes and eclass_derive_indexes. + */ + + foreach(lc, root->eq_sources) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_sources, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + } + } + + foreach(lc, root->eq_derives) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_derives, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + } + } +#endif } /* Attachments: [text/plain] sanity-check.txt (1.4K, ../../CAJ2pMkaaY4FgGSO2z9X_XOoqtLMUhyx6cPZFQbxmsjLgLpEd2Q@mail.gmail.com/2-sanity-check.txt) download | inline diff: diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 2da2a6c14e..334ea8d3bb 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -1751,6 +1751,48 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) bms_free(ec->ec_source_indexes); ec->ec_source_indexes = new_source_indexes; ec->ec_relids = replace_relid(ec->ec_relids, from, to); + +#ifdef USE_ASSERT_CHECKING + /* + * Sanity check: + * Verify that there are no missing references between RestrictInfos and + * our indexes, namely eclass_source_indexes and eclass_derive_indexes. + */ + + foreach(lc, root->eq_sources) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_sources, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + } + } + + foreach(lc, root->eq_derives) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_derives, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + } + } +#endif } /* [application/octet-stream] v23-0001-Speed-up-searches-for-child-EquivalenceMembers.patch (61.3K, ../../CAJ2pMkaaY4FgGSO2z9X_XOoqtLMUhyx6cPZFQbxmsjLgLpEd2Q@mail.gmail.com/3-v23-0001-Speed-up-searches-for-child-EquivalenceMembers.patch) download | inline diff: From a95566fa43f01a4589cc300fe1985bc86abab6fd Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:43:36 +0900 Subject: [PATCH v23 1/6] Speed up searches for child EquivalenceMembers Traditionally, child EquivalenceMembers were in the EquivalenceClass->ec_members. When we wanted to find some child members matching a request, we had to perform a linear search. This search became heavy when tables had many partitions, leading to much planning time. After this commit, child EquivalenceMembers no longer exist in ec_members. Instead, RelOptInfos have them. This change demonstrates a significant performance improvement in planning time. --- contrib/postgres_fdw/postgres_fdw.c | 29 +- src/backend/nodes/outfuncs.c | 2 + src/backend/optimizer/path/equivclass.c | 414 ++++++++++++++++++---- src/backend/optimizer/path/indxpath.c | 40 ++- src/backend/optimizer/path/pathkeys.c | 9 +- src/backend/optimizer/plan/analyzejoins.c | 14 +- src/backend/optimizer/plan/createplan.c | 57 +-- src/backend/optimizer/util/inherit.c | 14 + src/backend/optimizer/util/relnode.c | 88 +++++ src/include/nodes/pathnodes.h | 26 ++ src/include/optimizer/pathnode.h | 18 + src/include/optimizer/paths.h | 9 +- 12 files changed, 603 insertions(+), 117 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 142dcfc995..dbaefab1f6 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7773,13 +7773,27 @@ conversion_error_callback(void *arg) EquivalenceMember * find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) { - ListCell *lc; - PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + Relids top_parent_rel_relids; + List *members; + bool modified; + int i; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + + members = ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + EquivalenceMember *em = list_nth_node(EquivalenceMember, members, i); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_rel_relids)) + add_child_rel_equivalences_to_list(root, ec, em, + rel->relids, + &members, &modified); /* * Note we require !bms_is_empty, else we'd accept constant @@ -7791,6 +7805,8 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) is_foreign_expr(root, rel, em->em_expr)) return em; } + if (unlikely(modified)) + list_free(members); return NULL; } @@ -7844,9 +7860,8 @@ find_em_for_rel_target(PlannerInfo *root, EquivalenceClass *ec, if (em->em_is_const) continue; - /* Ignore child members */ - if (em->em_is_child) - continue; + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* Match if same expression (after stripping relabel) */ em_expr = em->em_expr; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 296ba84518..97ab5d3770 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -463,6 +463,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_opfamilies); WRITE_OID_FIELD(ec_collation); WRITE_NODE_FIELD(ec_members); + WRITE_NODE_FIELD(ec_norel_members); + WRITE_NODE_FIELD(ec_rel_members); WRITE_NODE_FIELD(ec_sources); WRITE_NODE_FIELD(ec_derives); WRITE_BITMAPSET_FIELD(ec_relids); diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index e86dfeaecd..d9eb4610ce 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -33,10 +33,14 @@ #include "utils/lsyscache.h" +static EquivalenceMember *make_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + EquivalenceMember *parent, + Oid datatype); static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, - EquivalenceMember *parent, Oid datatype); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, @@ -69,6 +73,12 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo); static JoinDomain *find_join_domain(PlannerInfo *root, Relids relids); +static void add_transformed_child_version(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel, + List **list, + bool *modified); static Bitmapset *get_eclass_indexes_for_relids(PlannerInfo *root, Relids relids); static Bitmapset *get_common_eclass_indexes(PlannerInfo *root, Relids relids1, @@ -342,6 +352,10 @@ process_equivalence(PlannerInfo *root, * be found. */ ec1->ec_members = list_concat(ec1->ec_members, ec2->ec_members); + ec1->ec_norel_members = list_concat(ec1->ec_norel_members, + ec2->ec_norel_members); + ec1->ec_rel_members = list_concat(ec1->ec_rel_members, + ec2->ec_rel_members); ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); @@ -355,6 +369,8 @@ process_equivalence(PlannerInfo *root, root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; + ec2->ec_norel_members = NIL; + ec2->ec_rel_members = NIL; ec2->ec_sources = NIL; ec2->ec_derives = NIL; ec2->ec_relids = NULL; @@ -374,7 +390,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); @@ -391,7 +407,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); @@ -412,6 +428,8 @@ process_equivalence(PlannerInfo *root, ec->ec_opfamilies = opfamilies; ec->ec_collation = collation; ec->ec_members = NIL; + ec->ec_norel_members = NIL; + ec->ec_rel_members = NIL; ec->ec_sources = list_make1(restrictinfo); ec->ec_derives = NIL; ec->ec_relids = NULL; @@ -423,9 +441,9 @@ process_equivalence(PlannerInfo *root, ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; em1 = add_eq_member(ec, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); em2 = add_eq_member(ec, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -511,11 +529,14 @@ canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation) } /* - * add_eq_member - build a new EquivalenceMember and add it to an EC + * make_eq_member + * + * Build a new EquivalenceMember without adding it to an EC. Note that child + * EquivalenceMembers should not be added to its parent EquivalenceClass. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) +make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) { EquivalenceMember *em = makeNode(EquivalenceMember); @@ -526,6 +547,8 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, em->em_datatype = datatype; em->em_jdomain = jdomain; em->em_parent = parent; + em->em_child_relids = NULL; + em->em_child_joinrel_relids = NULL; if (bms_is_empty(relids)) { @@ -546,8 +569,34 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, { ec->ec_relids = bms_add_members(ec->ec_relids, relids); } + + return em; +} + +/* + * add_eq_member - build a new EquivalenceMember and add it to an EC + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + ec->ec_members = lappend(ec->ec_members, em); + /* + * The exact set of relids in the expr for non-child EquivalenceMembers + * as what is given to us in 'relids' should be the same as the relids + * mentioned in the expression. See add_child_rel_equivalences. + */ + /* XXX We need PlannerInfo to use the following assertion */ + /* Assert(bms_equal(pull_varnos(root, (Node *) expr), relids)); */ + if (bms_is_empty(relids)) + ec->ec_norel_members = lappend(ec->ec_norel_members, em); + else + ec->ec_rel_members = lappend(ec->ec_rel_members, em); + return em; } @@ -599,6 +648,17 @@ get_eclass_for_sort_expr(PlannerInfo *root, EquivalenceMember *newem; ListCell *lc1; MemoryContext oldcontext; + Relids top_parent_rel; + + /* + * First, we translate the given Relids to their top-level parents. This is + * required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get child + * members. We can skip such translations if we now see top-level ones, + * i.e., when top_parent_rel is NULL. See the find_relids_top_parents()'s + * definition for more details. + */ + top_parent_rel = find_relids_top_parents(root, rel); /* * Ensure the expression exposes the correct type and collation. @@ -617,7 +677,9 @@ get_eclass_for_sort_expr(PlannerInfo *root, foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); - ListCell *lc2; + List *members; + bool modified; + int i; /* * Never match to a volatile EC, except when we are looking at another @@ -632,16 +694,35 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (!equal(opfamilies, cur_ec->ec_opfamilies)) continue; - foreach(lc2, cur_ec->ec_members) + /* + * When we have to see child EquivalenceMembers, we get and add them to + * 'members'. We cannot use foreach() because the 'members' may be + * modified during iteration. + */ + members = cur_ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); + EquivalenceMember *cur_em = list_nth_node(EquivalenceMember, members, i); + + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them. + */ + if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel)) + add_child_rel_equivalences_to_list(root, cur_ec, cur_em, rel, + &members, &modified); /* * Ignore child members unless they match the request. */ - if (cur_em->em_is_child && - !bms_equal(cur_em->em_relids, rel)) - continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(!cur_em->em_is_child || bms_equal(cur_em->em_relids, rel)); /* * Match constants only within the same JoinDomain (see @@ -654,6 +735,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, equal(expr, cur_em->em_expr)) return cur_ec; /* Match! */ } + if (unlikely(modified)) + list_free(members); } /* No match; does caller want a NULL result? */ @@ -671,6 +754,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = collation; newec->ec_members = NIL; + newec->ec_norel_members = NIL; + newec->ec_rel_members = NIL; newec->ec_sources = NIL; newec->ec_derives = NIL; newec->ec_relids = NULL; @@ -691,7 +776,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, expr_relids = pull_varnos(root, (Node *) expr); newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, NULL, opcintype); + jdomain, opcintype); /* * add_eq_member doesn't check for volatile functions, set-returning @@ -757,19 +842,28 @@ get_eclass_for_sort_expr(PlannerInfo *root, * Child EC members are ignored unless they belong to given 'relids'. */ EquivalenceMember * -find_ec_member_matching_expr(EquivalenceClass *ec, +find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, Expr *expr, Relids relids) { - ListCell *lc; + Relids top_parent_relids; + List *members; + bool modified; + int i; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); /* We ignore binary-compatible relabeling on both ends */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + EquivalenceMember *em = list_nth_node(EquivalenceMember, members, i); Expr *emexpr; /* @@ -779,6 +873,12 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + add_child_rel_equivalences_to_list(root, ec, em, relids, + &members, &modified); + /* * Ignore child members unless they belong to the requested rel. */ @@ -796,6 +896,8 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (equal(emexpr, expr)) return em; } + if (unlikely(modified)) + list_free(members); return NULL; } @@ -828,11 +930,20 @@ find_computable_ec_member(PlannerInfo *root, Relids relids, bool require_parallel_safe) { - ListCell *lc; + Relids top_parent_relids; + List *members; + bool modified; + int i; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = ec->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + EquivalenceMember *em = list_nth_node(EquivalenceMember, members, i); List *exprvars; ListCell *lc2; @@ -843,6 +954,12 @@ find_computable_ec_member(PlannerInfo *root, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + add_child_rel_equivalences_to_list(root, ec, em, relids, + &members, &modified); + /* * Ignore child members unless they belong to the requested rel. */ @@ -876,6 +993,8 @@ find_computable_ec_member(PlannerInfo *root, return em; /* found usable expression */ } + if (unlikely(modified)) + list_free(members); return NULL; } @@ -939,7 +1058,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, { Expr *targetexpr = (Expr *) lfirst(lc); - em = find_ec_member_matching_expr(ec, targetexpr, rel->relids); + em = find_ec_member_matching_expr(root, ec, targetexpr, rel->relids); if (!em) continue; @@ -1138,7 +1257,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, * machinery might be able to exclude relations on the basis of generated * "var = const" equalities, but "var = param" won't work for that. */ - foreach(lc, ec->ec_members) + foreach(lc, ec->ec_norel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); @@ -1222,7 +1341,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, prev_ems = (EquivalenceMember **) palloc0(root->simple_rel_array_size * sizeof(EquivalenceMember *)); - foreach(lc, ec->ec_members) + foreach(lc, ec->ec_rel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); int relid; @@ -1559,7 +1678,13 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *new_members = NIL; List *outer_members = NIL; List *inner_members = NIL; - ListCell *lc1; + Relids top_parent_join_relids; + List *members; + bool modified; + int i; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_join_relids = find_relids_top_parents(root, join_relids); /* * First, scan the EC to identify member values that are computable at the @@ -1570,9 +1695,19 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * as well as to at least one input member, plus enforce at least one * outer-rel member equal to at least one inner-rel member. */ - foreach(lc1, ec->ec_members) + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = ec->ec_rel_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + EquivalenceMember *cur_em = list_nth_node(EquivalenceMember, members, i); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + bms_is_subset(cur_em->em_relids, top_parent_join_relids)) + add_child_rel_equivalences_to_list(root, ec, cur_em, join_relids, + &members, &modified); /* * We don't need to check explicitly for child EC members. This test @@ -1589,6 +1724,8 @@ generate_join_implied_equalities_normal(PlannerInfo *root, else new_members = lappend(new_members, cur_em); } + if (unlikely(modified)) + list_free(members); /* * First, select the joinclause if needed. We can equate any one outer @@ -1606,6 +1743,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, Oid best_eq_op = InvalidOid; int best_score = -1; RestrictInfo *rinfo; + ListCell *lc1; foreach(lc1, outer_members) { @@ -1680,6 +1818,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *old_members = list_concat(outer_members, inner_members); EquivalenceMember *prev_em = NULL; RestrictInfo *rinfo; + ListCell *lc1; /* For now, arbitrarily take the first old_member as the one to use */ if (old_members) @@ -2177,7 +2316,7 @@ reconsider_outer_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo, * constant before we can decide to throw away the outer-join clause. */ match = false; - foreach(lc2, cur_ec->ec_members) + foreach(lc2, cur_ec->ec_norel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); Oid eq_op; @@ -2285,7 +2424,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) * the COALESCE arguments? */ match = false; - foreach(lc2, cur_ec->ec_members) + foreach(lc2, cur_ec->ec_rel_members) { coal_em = (EquivalenceMember *) lfirst(lc2); Assert(!coal_em->em_is_child); /* no children yet */ @@ -2330,7 +2469,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) * decide to throw away the outer-join clause. */ matchleft = matchright = false; - foreach(lc2, cur_ec->ec_members) + foreach(lc2, cur_ec->ec_norel_members) { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); Oid eq_op; @@ -2385,6 +2524,10 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) if (matchleft && matchright) { cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); + Assert(!bms_is_empty(coal_em->em_relids)); + /* XXX performance of list_delete_ptr()?? */ + cur_ec->ec_rel_members = list_delete_ptr(cur_ec->ec_rel_members, + coal_em); return true; } @@ -2455,8 +2598,8 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); if (equal(item1, em->em_expr)) item1member = true; else if (equal(item2, em->em_expr)) @@ -2523,13 +2666,13 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, continue; /* Note: it seems okay to match to "broken" eclasses here */ - foreach(lc2, ec->ec_members) + foreach(lc2, ec->ec_rel_members) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); Var *var; - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* EM must be a Var, possibly with RelabelType */ var = (Var *) em->em_expr; @@ -2626,6 +2769,7 @@ add_child_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_rel->top_parent_relids; Relids child_relids = child_rel->relids; int i; + ListCell *lc; /* * EC merging should be complete already, so we can use the parent rel's @@ -2638,7 +2782,6 @@ add_child_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2651,15 +2794,9 @@ add_child_rel_equivalences(PlannerInfo *root, /* Sanity check eclass_indexes only contain ECs for parent_rel */ Assert(bms_is_subset(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_rel_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2672,8 +2809,8 @@ add_child_rel_equivalences(PlannerInfo *root, * combinations of children. (But add_child_join_rel_equivalences * may add targeted combinations for partitionwise-join purposes.) */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * Consider only members that reference and can be computed at @@ -2689,6 +2826,7 @@ add_child_rel_equivalences(PlannerInfo *root, /* OK, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_rel->reloptkind == RELOPT_BASEREL) { @@ -2718,9 +2856,20 @@ add_child_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, + child_em); + + /* + * We save the knowledge that 'child_em' can be translated from + * 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + cur_em->em_child_relids = bms_add_member(cur_em->em_child_relids, + child_rel->relid); /* Record this EC index for the child rel */ child_rel->eclass_indexes = bms_add_member(child_rel->eclass_indexes, i); @@ -2749,6 +2898,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; MemoryContext oldcontext; + ListCell *lc; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2770,7 +2920,6 @@ add_child_join_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(matching_ecs, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2783,15 +2932,9 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Sanity check on get_eclass_indexes_for_relids result */ Assert(bms_overlap(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_rel_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2800,8 +2943,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, * We consider only original EC members here, not * already-transformed child members. */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * We may ignore expressions that reference a single baserel, @@ -2816,6 +2959,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Yes, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_joinrel->reloptkind == RELOPT_JOINREL) { @@ -2846,9 +2990,21 @@ add_child_join_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_joinrel->eclass_child_members = + lappend(child_joinrel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated from + * 'child_joinrel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + cur_em->em_child_joinrel_relids = + bms_add_member(cur_em->em_child_joinrel_relids, + child_joinrel->join_rel_list_index); } } } @@ -2856,6 +3012,106 @@ add_child_join_rel_equivalences(PlannerInfo *root, MemoryContextSwitchTo(oldcontext); } +/* + * add_transformed_child_version + * Add a transformed EquivalenceMember referencing the given child_rel to + * the list. + * + * This function is expected to be called only from + * add_child_rel_equivalences_to_list(). + */ +static void +add_transformed_child_version(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel, + List **list, + bool *modified) +{ + ListCell *lc; + + foreach(lc, child_rel->eclass_child_members) + { + EquivalenceMember *child_em = lfirst_node(EquivalenceMember, lc); + + /* Skip unwanted EquivalenceMembers */ + if (child_em->em_parent != parent_em) + continue; + + /* + * If this is the first time the given list has been modified, we need to + * make a copy of it. + */ + if (!*modified) + { + *list = list_copy(*list); + *modified = true; + } + + /* Add this child EquivalenceMember to the list */ + *list = lappend(*list, child_em); + } +} + +/* + * add_child_rel_equivalences_to_list + * Add transformed EquivalenceMembers referencing child rels in the given + * child_relids to the list. + * + * The transformation is done in add_transformed_child_version(). + * + * This function will not change the original *list but will always make a copy + * of it when we need to add transformed members. *modified will be true if the + * *list is replaced with a newly allocated one. In such a case, the caller can + * (or must) free the *list. + */ +void +add_child_rel_equivalences_to_list(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids, + List **list, + bool *modified) +{ + int i; + Relids matching_relids; + + /* The given EquivalenceMember should be parent */ + Assert(!parent_em->em_is_child); + + /* + * First, we translate simple rels. + */ + matching_relids = bms_intersect(parent_em->em_child_relids, + child_relids); + i = -1; + while ((i = bms_next_member(matching_relids, i)) >= 0) + { + /* Add transformed child version for this child rel */ + RelOptInfo *child_rel = root->simple_rel_array[i]; + + Assert(child_rel != NULL); + add_transformed_child_version(root, ec, parent_em, child_rel, + list, modified); + } + bms_free(matching_relids); + + /* + * Next, we try to translate join rels. + */ + i = -1; + while ((i = bms_next_member(parent_em->em_child_joinrel_relids, i)) >= 0) + { + /* Add transformed child version for this child join rel */ + RelOptInfo *child_joinrel = + list_nth_node(RelOptInfo, root->join_rel_list, i); + + Assert(child_joinrel != NULL); + add_transformed_child_version(root, ec, parent_em, child_joinrel, + list, modified); + } +} + /* * generate_implied_equalities_for_column @@ -2889,7 +3145,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, { List *result = NIL; bool is_child_rel = (rel->reloptkind == RELOPT_OTHER_MEMBER_REL); - Relids parent_relids; + Relids parent_relids, top_parent_rel_relids; int i; /* Should be OK to rely on eclass_indexes */ @@ -2898,6 +3154,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, /* Indexes are available only on base or "other" member relations. */ Assert(IS_SIMPLE_REL(rel)); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + /* If it's a child rel, we'll need to know what its parent(s) are */ if (is_child_rel) parent_relids = find_childrel_parents(root, rel); @@ -2910,6 +3169,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); EquivalenceMember *cur_em; ListCell *lc2; + List *members; + bool modified; + int j; /* Sanity check eclass_indexes only contain ECs for rel */ Assert(is_child_rel || bms_is_subset(rel->relids, cur_ec->ec_relids)); @@ -2931,15 +3193,25 @@ generate_implied_equalities_for_column(PlannerInfo *root, * corner cases, so for now we live with just reporting the first * match. See also get_eclass_for_sort_expr.) */ + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = cur_ec->ec_rel_members; + modified = false; cur_em = NULL; - foreach(lc2, cur_ec->ec_members) + for (j = 0; j < list_length(members); j++) { - cur_em = (EquivalenceMember *) lfirst(lc2); + cur_em = list_nth_node(EquivalenceMember, members, j); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel_relids)) + add_child_rel_equivalences_to_list(root, cur_ec, cur_em, rel->relids, + &members, &modified); if (bms_equal(cur_em->em_relids, rel->relids) && callback(root, rel, cur_ec, cur_em, callback_arg)) break; cur_em = NULL; } + if (unlikely(modified)) + list_free(members); if (!cur_em) continue; @@ -2954,8 +3226,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, Oid eq_op; RestrictInfo *rinfo; - if (other_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!other_em->em_is_child); /* Make sure it'll be a join to a different rel */ if (other_em == cur_em || @@ -3173,8 +3445,8 @@ eclass_useful_for_merging(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); if (!bms_overlap(cur_em->em_relids, relids)) return true; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 32c6a8bbdc..1f7438a619 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -184,7 +184,7 @@ static IndexClause *expand_indexqual_rowcompare(PlannerInfo *root, IndexOptInfo *index, Oid expr_op, bool var_on_left); -static void match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +static void match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p); static Expr *match_clause_to_ordering_op(IndexOptInfo *index, @@ -980,7 +980,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * query_pathkeys will allow an incremental sort to be considered on * the index's partially sorted results. */ - match_pathkeys_to_index(index, root->query_pathkeys, + match_pathkeys_to_index(root, index, root->query_pathkeys, &orderbyclauses, &orderbyclausecols); if (list_length(root->query_pathkeys) == list_length(orderbyclauses)) @@ -3071,12 +3071,16 @@ expand_indexqual_rowcompare(PlannerInfo *root, * item in the given 'pathkeys' list. */ static void -match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p) { + Relids top_parent_index_relids; ListCell *lc1; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_index_relids = find_relids_top_parents(root, index->rel->relids); + *orderby_clauses_p = NIL; /* set default results */ *clause_columns_p = NIL; @@ -3088,7 +3092,9 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, { PathKey *pathkey = (PathKey *) lfirst(lc1); bool found = false; - ListCell *lc2; + List *members; + bool modified; + int i; /* Pathkey must request default sort order for the target opfamily */ @@ -3108,15 +3114,33 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, * be considered to match more than one pathkey list, which is OK * here. See also get_eclass_for_sort_expr.) */ - foreach(lc2, pathkey->pk_eclass->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + members = pathkey->pk_eclass->ec_members; + modified = false; + for (i = 0; i < list_length(members); i++) { - EquivalenceMember *member = (EquivalenceMember *) lfirst(lc2); + EquivalenceMember *member = list_nth_node(EquivalenceMember, members, i); int indexcol; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + bms_equal(member->em_relids, top_parent_index_relids)) + add_child_rel_equivalences_to_list(root, pathkey->pk_eclass, member, + index->rel->relids, + &members, &modified); + /* No possibility of match if it references other relations */ - if (!bms_equal(member->em_relids, index->rel->relids)) + if (member->em_is_child || + !bms_equal(member->em_relids, index->rel->relids)) continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(bms_equal(member->em_relids, index->rel->relids)); + /* * We allow any column of the index to match each pathkey; they * don't have to match left-to-right as you might expect. This is @@ -3145,6 +3169,8 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, if (found) /* don't want to look at remaining members */ break; } + if (unlikely(modified)) + list_free(members); /* * Return the matches found so far when this pathkey couldn't be diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index ca94a31f71..aac64c4124 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -952,8 +952,8 @@ convert_subquery_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid sub_expr_coll = sub_eclass->ec_collation; ListCell *k; - if (sub_member->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!sub_member->em_is_child); foreach(k, subquery_tlist) { @@ -1479,8 +1479,11 @@ select_outer_pathkeys_for_merge(PlannerInfo *root, { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); + /* Potential future join partner? */ - if (!em->em_is_const && !em->em_is_child && + if (!em->em_is_const && !bms_overlap(em->em_relids, joinrel->relids)) score++; } diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 7dcb74572a..f0735f3c41 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -61,7 +61,7 @@ static void remove_leftjoinrel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid); -static void remove_rel_from_eclass(EquivalenceClass *ec, +static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static bool rel_supports_distinctness(PlannerInfo *root, RelOptInfo *rel); @@ -580,7 +580,7 @@ remove_leftjoinrel_from_query(PlannerInfo *root, int relid, if (bms_is_member(relid, ec->ec_relids) || bms_is_member(ojrelid, ec->ec_relids)) - remove_rel_from_eclass(ec, relid, ojrelid); + remove_rel_from_eclass(root, ec, relid, ojrelid); } /* @@ -665,7 +665,8 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) * level(s). */ static void -remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) +remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, + int ojrelid) { ListCell *lc; @@ -689,7 +690,14 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) cur_em->em_relids = bms_del_member(cur_em->em_relids, relid); cur_em->em_relids = bms_del_member(cur_em->em_relids, ojrelid); if (bms_is_empty(cur_em->em_relids)) + { ec->ec_members = foreach_delete_current(ec->ec_members, lc); + /* XXX performance of list_delete_ptr()?? */ + ec->ec_norel_members = list_delete_ptr(ec->ec_norel_members, + cur_em); + ec->ec_rel_members = list_delete_ptr(ec->ec_rel_members, + cur_em); + } } } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index ca619eab94..dce42ada81 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -260,7 +260,9 @@ static IncrementalSort *make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); -static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -269,9 +271,11 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); -static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Sort *make_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids); -static IncrementalSort *make_incrementalsort_from_pathkeys(Plan *lefttree, +static IncrementalSort *make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, int nPresortedCols); static Sort *make_sort_from_groupcols(List *groupcls, AttrNumber *grpColIdx, @@ -293,7 +297,7 @@ static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); static Unique *make_unique_from_sortclauses(Plan *lefttree, List *distinctList); -static Unique *make_unique_from_pathkeys(Plan *lefttree, +static Unique *make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, int numCols); static Gather *make_gather(List *qptlist, List *qpqual, int nworkers, int rescan_param, bool single_copy, Plan *subplan); @@ -1280,7 +1284,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * function result; it must be the same plan node. However, we then * need to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys((Plan *) plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, (Plan *) plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1324,7 +1328,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * don't need an explicit sort, to make sure they are returning * the same sort key columns the Append expects. */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, nodeSortColIdx, false, @@ -1465,7 +1469,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * function result; it must be the same plan node. However, we then need * to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys(plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1496,7 +1500,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, subplan = create_plan_recurse(root, subpath, CP_EXACT_TLIST); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, node->sortColIdx, false, @@ -1975,7 +1979,7 @@ create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) Assert(pathkeys != NIL); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, best_path->subpath->parent->relids, gm_plan->sortColIdx, false, @@ -2194,7 +2198,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) * relids. Thus, if this sort path is based on a child relation, we must * pass its relids. */ - plan = make_sort_from_pathkeys(subplan, best_path->path.pathkeys, + plan = make_sort_from_pathkeys(root, subplan, best_path->path.pathkeys, IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -2218,7 +2222,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, /* See comments in create_sort_plan() above */ subplan = create_plan_recurse(root, best_path->spath.subpath, flags | CP_SMALL_TLIST); - plan = make_incrementalsort_from_pathkeys(subplan, + plan = make_incrementalsort_from_pathkeys(root, subplan, best_path->spath.path.pathkeys, IS_OTHER_REL(best_path->spath.subpath->parent) ? best_path->spath.path.parent->relids : NULL, @@ -2287,7 +2291,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag subplan = create_plan_recurse(root, best_path->subpath, flags | CP_LABEL_TLIST); - plan = make_unique_from_pathkeys(subplan, + plan = make_unique_from_pathkeys(root, subplan, best_path->path.pathkeys, best_path->numkeys); @@ -4498,7 +4502,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->outersortkeys) { Relids outer_relids = outer_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(outer_plan, + Sort *sort = make_sort_from_pathkeys(root, outer_plan, best_path->outersortkeys, outer_relids); @@ -4512,7 +4516,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->innersortkeys) { Relids inner_relids = inner_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(inner_plan, + Sort *sort = make_sort_from_pathkeys(root, inner_plan, best_path->innersortkeys, inner_relids); @@ -6133,7 +6137,7 @@ make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, * or a Result stacked atop lefttree). */ static Plan * -prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -6200,7 +6204,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ @@ -6228,7 +6232,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr already in tlist */ @@ -6244,7 +6248,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, /* * No matching tlist item; look for a computable expression. */ - em = find_computable_ec_member(NULL, ec, tlist, relids, false); + em = find_computable_ec_member(root, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; @@ -6315,7 +6319,8 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ static Sort * -make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) +make_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + Relids relids) { int numsortkeys; AttrNumber *sortColIdx; @@ -6324,7 +6329,7 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6350,8 +6355,9 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) * 'nPresortedCols' is the number of presorted columns in input tuples */ static IncrementalSort * -make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, - Relids relids, int nPresortedCols) +make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, + List *pathkeys, Relids relids, + int nPresortedCols) { int numsortkeys; AttrNumber *sortColIdx; @@ -6360,7 +6366,7 @@ make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6717,7 +6723,8 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) * as above, but use pathkeys to identify the sort columns and semantics */ static Unique * -make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) +make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + int numCols) { Unique *node = makeNode(Unique); Plan *plan = &node->plan; @@ -6780,7 +6787,7 @@ make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) foreach(j, plan->targetlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, NULL); + em = find_ec_member_matching_expr(root, ec, tle->expr, NULL); if (em) { /* found expr already in tlist */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 5c7acf8a90..3501fbbeed 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -461,6 +461,7 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, RelationGetRelid(parentrel); Oid childOID = RelationGetRelid(childrel); RangeTblEntry *childrte; + Index topParentRTindex; Index childRTindex; AppendRelInfo *appinfo; TupleDesc child_tupdesc; @@ -568,6 +569,19 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, Assert(root->append_rel_array[childRTindex] == NULL); root->append_rel_array[childRTindex] = appinfo; + /* + * Find a top parent rel's index and save it to top_parent_relid_array. + */ + if (root->top_parent_relid_array == NULL) + root->top_parent_relid_array = + (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); + Assert(root->top_parent_relid_array[childRTindex] == 0); + topParentRTindex = parentRTindex; + while (root->append_rel_array[topParentRTindex] != NULL && + root->append_rel_array[topParentRTindex]->parent_relid != 0) + topParentRTindex = root->append_rel_array[topParentRTindex]->parent_relid; + root->top_parent_relid_array[childRTindex] = topParentRTindex; + /* * Build a PlanRowMark if parent is marked FOR UPDATE/SHARE. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 22d01cef5b..b8185d0dda 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -123,11 +123,14 @@ setup_simple_rel_arrays(PlannerInfo *root) if (root->append_rel_list == NIL) { root->append_rel_array = NULL; + root->top_parent_relid_array = NULL; return; } root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); + root->top_parent_relid_array = (Index *) + palloc0(size * sizeof(Index)); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -148,6 +151,27 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array[child_relid] = appinfo; } + + /* + * Find a top parent rel's relid for each AppendRelInfo. We cannot do this + * in the last foreach loop because there may be multi-level parent-child + * relations. + */ + for (int i = 0; i < size; i++) + { + int top_parent_relid; + + if (root->append_rel_array[i] == NULL) + continue; + + top_parent_relid = root->append_rel_array[i]->parent_relid; + + while (root->append_rel_array[top_parent_relid] != NULL && + root->append_rel_array[top_parent_relid]->parent_relid != 0) + top_parent_relid = root->append_rel_array[top_parent_relid]->parent_relid; + + root->top_parent_relid_array[i] = top_parent_relid; + } } /* @@ -175,11 +199,23 @@ expand_planner_arrays(PlannerInfo *root, int add_size) repalloc0_array(root->simple_rte_array, RangeTblEntry *, root->simple_rel_array_size, new_size); if (root->append_rel_array) + { root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); + root->top_parent_relid_array = + repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + } else + { root->append_rel_array = palloc0_array(AppendRelInfo *, new_size); + /* + * We do not allocate top_parent_relid_array here so that + * find_relids_top_parents() can early find all of the given Relids are + * top-level. + */ + root->top_parent_relid_array = NULL; + } root->simple_rel_array_size = new_size; } @@ -233,6 +269,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; + rel->eclass_child_members = NIL; rel->serverid = InvalidOid; if (rte->rtekind == RTE_RELATION) { @@ -621,6 +658,12 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) /* GEQO requires us to append the new joinrel to the end of the list! */ root->join_rel_list = lappend(root->join_rel_list, joinrel); + /* + * Store the index of this joinrel to use in + * add_child_join_rel_equivalences(). + */ + joinrel->join_rel_list_index = list_length(root->join_rel_list) - 1; + /* store it into the auxiliary hashtable if there is one. */ if (root->join_rel_hash) { @@ -732,6 +775,7 @@ build_join_rel(PlannerInfo *root, joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -928,6 +972,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -1492,6 +1537,7 @@ fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids) upperrel->cheapest_total_path = NULL; upperrel->cheapest_unique_path = NULL; upperrel->cheapest_parameterized_paths = NIL; + upperrel->eclass_child_members = NIL; /* XXX Is this required? */ root->upper_rels[kind] = lappend(root->upper_rels[kind], upperrel); @@ -1532,6 +1578,48 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } +/* + * find_relids_top_parents_slow + * Slow path of find_relids_top_parents() macro. + * + * Do not call this directly; use the macro instead. See the macro comment for + * more details. + */ +Relids +find_relids_top_parents_slow(PlannerInfo *root, Relids relids) +{ + Index *top_parent_relid_array = root->top_parent_relid_array; + Relids result; + bool is_top_parent; + int i; + + /* This function should be called in the slow path */ + Assert(top_parent_relid_array != NULL); + + result = NULL; + is_top_parent = true; + i = -1; + while ((i = bms_next_member(relids, i)) >= 0) + { + int top_parent_relid = (int) top_parent_relid_array[i]; + + if (top_parent_relid == 0) + top_parent_relid = i; /* 'i' has no parents, so add itself */ + else if (top_parent_relid != i) + is_top_parent = false; + result = bms_add_member(result, top_parent_relid); + } + + if (is_top_parent) + { + bms_free(result); + return NULL; + } + + return result; +} + + /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index b9713ec9aa..8c55a2bcf0 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -245,6 +245,14 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * append_rel_array is the same length as simple_rel_array and holds the + * top-level parent indexes of the corresponding rels within + * simple_rel_array. The element can be zero if the rel has no parents, + * i.e., is itself in a top-level. + */ + Index *top_parent_relid_array pg_node_attr(read_write_ignore); + /* * all_baserels is a Relids set of all base relids (but not joins or * "other" rels) in the query. This is computed in deconstruct_jointree. @@ -936,6 +944,17 @@ typedef struct RelOptInfo /* Bitmask of optional features supported by the table AM */ uint32 amflags; + /* + * information about a join rel + */ + /* index in root->join_rel_list of this rel */ + Index join_rel_list_index; + + /* + * EquivalenceMembers referencing this rel + */ + List *eclass_child_members; + /* * Information about foreign tables and foreign joins */ @@ -1368,6 +1387,10 @@ typedef struct EquivalenceClass List *ec_opfamilies; /* btree operator family OIDs */ Oid ec_collation; /* collation, if datatypes are collatable */ List *ec_members; /* list of EquivalenceMembers */ + List *ec_norel_members; /* list of EquivalenceMembers whose + * em_relids is empty */ + List *ec_rel_members; /* list of EquivalenceMembers whose + * em_relids is not empty */ List *ec_sources; /* list of generating RestrictInfos */ List *ec_derives; /* list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except @@ -1422,6 +1445,9 @@ typedef struct EquivalenceMember bool em_is_child; /* derived version for a child relation? */ Oid em_datatype; /* the "nominal type" used by the opfamily */ JoinDomain *em_jdomain; /* join domain containing the source clause */ + Relids em_child_relids; /* all relids of child rels */ + Bitmapset *em_child_joinrel_relids; /* indexes in root->join_rel_list of + join rel children */ /* if em_is_child is true, this links to corresponding EM for top parent */ struct EquivalenceMember *em_parent pg_node_attr(read_write_ignore); } EquivalenceMember; diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index c43d97b48a..e61f2e3529 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -324,6 +324,24 @@ extern Relids min_join_parameterization(PlannerInfo *root, extern RelOptInfo *fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids); extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); + +/* + * find_relids_top_parents + * Compute the set of top-parent relids of rel. + * + * Replaces all Relids appearing in the given 'relids' as their top-level + * parents. The result will be NULL if and only if all of the given relids are + * top-level. + * + * The motivation for having this feature as a macro rather than a function is + * that Relids are top-level in most cases. We can quickly determine when + * root->top_parent_relid_array is NULL. + */ +#define find_relids_top_parents(root, relids) \ + (likely((root)->top_parent_relid_array == NULL) \ + ? NULL : find_relids_top_parents_slow(root, relids)) +extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); + extern ParamPathInfo *get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, Relids required_outer); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index efd4abc28f..9bc0c36b93 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -136,7 +136,8 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Index sortref, Relids rel, bool create_it); -extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec, +extern EquivalenceMember *find_ec_member_matching_expr(PlannerInfo *root, + EquivalenceClass *ec, Expr *expr, Relids relids); extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root, @@ -173,6 +174,12 @@ extern void add_child_join_rel_equivalences(PlannerInfo *root, AppendRelInfo **appinfos, RelOptInfo *parent_joinrel, RelOptInfo *child_joinrel); +extern void add_child_rel_equivalences_to_list(PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids, + List **list, + bool *modified); extern List *generate_implied_equalities_for_column(PlannerInfo *root, RelOptInfo *rel, ec_matches_callback_type callback, -- 2.42.0.windows.2 [application/octet-stream] v23-0002-Introduce-indexes-for-RestrictInfo.patch (31.6K, ../../CAJ2pMkaaY4FgGSO2z9X_XOoqtLMUhyx6cPZFQbxmsjLgLpEd2Q@mail.gmail.com/4-v23-0002-Introduce-indexes-for-RestrictInfo.patch) download | inline diff: From f4f0c7ade55cc5f0d3fc2bc62934710b8e12fa8d Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:44:25 +0900 Subject: [PATCH v23 2/6] Introduce indexes for RestrictInfo This change was picked up from v19. Author: David Rowley <[email protected]> and me --- src/backend/nodes/outfuncs.c | 6 +- src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/path/equivclass.c | 338 +++++++++++++++++++--- src/backend/optimizer/plan/analyzejoins.c | 8 +- src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/inherit.c | 7 + src/include/nodes/parsenodes.h | 6 + src/include/nodes/pathnodes.h | 47 ++- src/include/optimizer/paths.h | 15 +- 11 files changed, 383 insertions(+), 53 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 97ab5d3770..98f824ac53 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -465,8 +465,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_members); WRITE_NODE_FIELD(ec_norel_members); WRITE_NODE_FIELD(ec_rel_members); - WRITE_NODE_FIELD(ec_sources); - WRITE_NODE_FIELD(ec_derives); + WRITE_BITMAPSET_FIELD(ec_source_indexes); + WRITE_BITMAPSET_FIELD(ec_derive_indexes); WRITE_BITMAPSET_FIELD(ec_relids); WRITE_BOOL_FIELD(ec_has_const); WRITE_BOOL_FIELD(ec_has_volatile); @@ -569,6 +569,8 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(inh); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); + WRITE_BITMAPSET_FIELD(eclass_source_indexes); + WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 1624b34581..deebfaf994 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -574,6 +574,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); + READ_BITMAPSET_FIELD(eclass_source_indexes); + READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 8b76e98529..ae64b169f8 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5784,7 +5784,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, if (ec && ec->ec_has_const) { EquivalenceMember *em = fkinfo->fk_eclass_member[i]; - RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + RestrictInfo *rinfo = find_derived_clause_for_ec_member(root, + ec, em); if (rinfo) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index d9eb4610ce..d1019315ea 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -42,6 +42,10 @@ static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, Oid datatype); +static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); +static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); @@ -321,7 +325,6 @@ process_equivalence(PlannerInfo *root, /* If case 1, nothing to do, except add to sources */ if (ec1 == ec2) { - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -332,6 +335,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); return true; } @@ -356,8 +361,10 @@ process_equivalence(PlannerInfo *root, ec2->ec_norel_members); ec1->ec_rel_members = list_concat(ec1->ec_rel_members, ec2->ec_rel_members); - ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); - ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); + ec1->ec_source_indexes = bms_join(ec1->ec_source_indexes, + ec2->ec_source_indexes); + ec1->ec_derive_indexes = bms_join(ec1->ec_derive_indexes, + ec2->ec_derive_indexes); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); ec1->ec_has_const |= ec2->ec_has_const; /* can't need to set has_volatile */ @@ -371,10 +378,9 @@ process_equivalence(PlannerInfo *root, ec2->ec_members = NIL; ec2->ec_norel_members = NIL; ec2->ec_rel_members = NIL; - ec2->ec_sources = NIL; - ec2->ec_derives = NIL; + ec2->ec_source_indexes = NULL; + ec2->ec_derive_indexes = NULL; ec2->ec_relids = NULL; - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -385,13 +391,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec1) { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, jdomain, item2_type); - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -402,13 +409,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec2) { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, jdomain, item1_type); - ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -419,6 +427,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec2, restrictinfo); } else { @@ -430,8 +440,8 @@ process_equivalence(PlannerInfo *root, ec->ec_members = NIL; ec->ec_norel_members = NIL; ec->ec_rel_members = NIL; - ec->ec_sources = list_make1(restrictinfo); - ec->ec_derives = NIL; + ec->ec_source_indexes = NULL; + ec->ec_derive_indexes = NULL; ec->ec_relids = NULL; ec->ec_has_const = false; ec->ec_has_volatile = false; @@ -453,6 +463,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec, restrictinfo); } return true; @@ -600,6 +612,50 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, return em; } +/* + * add_eq_source - add 'rinfo' in eq_sources for this 'ec' + */ +static void +add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int source_idx = list_length(root->eq_sources); + int i; + + ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); + root->eq_sources = lappend(root->eq_sources, rinfo); + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + source_idx); + } +} + +/* + * add_eq_derive - add 'rinfo' in eq_derives for this 'ec' + */ +static void +add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int derive_idx = list_length(root->eq_derives); + int i; + + ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); + root->eq_derives = lappend(root->eq_derives, rinfo); + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + derive_idx); + } +} + /* * get_eclass_for_sort_expr @@ -756,8 +812,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_members = NIL; newec->ec_norel_members = NIL; newec->ec_rel_members = NIL; - newec->ec_sources = NIL; - newec->ec_derives = NIL; + newec->ec_source_indexes = NULL; + newec->ec_derive_indexes = NULL; newec->ec_relids = NULL; newec->ec_has_const = false; newec->ec_has_volatile = contain_volatile_functions((Node *) expr); @@ -1145,7 +1201,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources or ec_derives for matches. It doesn't really + * don't search eq_sources or eq_derives for matches. It doesn't really * seem worth the trouble to do so. */ void @@ -1234,6 +1290,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *const_em = NULL; ListCell *lc; + int i; /* * In the trivial case where we just had one "var = const" clause, push @@ -1243,9 +1300,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, * equivalent to the old one. */ if (list_length(ec->ec_members) == 2 && - list_length(ec->ec_sources) == 1) + bms_get_singleton_member(ec->ec_source_indexes, &i)) { - RestrictInfo *restrictinfo = (RestrictInfo *) linitial(ec->ec_sources); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, root->eq_sources, i); distribute_restrictinfo_to_rels(root, restrictinfo); return; @@ -1303,9 +1360,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the correct - * markings for a mergejoinable clause, and save it in ec_derives. (We + * markings for a mergejoinable clause, and save it in eq_derives. (We * will not re-use such clauses directly, but selectivity estimation - * may consult the list later. Note that this use of ec_derives does + * may consult the list later. Note that this use of eq_derives does * not overlap with its use for join clauses, since we never generate * join clauses from an ec_has_const eclass.) */ @@ -1315,7 +1372,8 @@ generate_base_implied_equalities_const(PlannerInfo *root, rinfo->left_ec = rinfo->right_ec = ec; rinfo->left_em = cur_em; rinfo->right_em = const_em; - ec->ec_derives = lappend(ec->ec_derives, rinfo); + + add_eq_derive(root, ec, rinfo); } } } @@ -1381,7 +1439,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the * correct markings for a mergejoinable clause. We don't put it - * in ec_derives however; we don't currently need to re-find such + * in eq_derives however; we don't currently need to re-find such * clauses, and we don't want to clutter that list with non-join * clauses. */ @@ -1437,11 +1495,12 @@ static void generate_base_implied_equalities_broken(PlannerInfo *root, EquivalenceClass *ec) { - ListCell *lc; + int i = -1; - foreach(lc, ec->ec_sources) + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); if (ec->ec_has_const || bms_membership(restrictinfo->required_relids) != BMS_MULTIPLE) @@ -1482,11 +1541,11 @@ generate_base_implied_equalities_broken(PlannerInfo *root, * Because the same join clauses are likely to be needed multiple times as * we consider different join paths, we avoid generating multiple copies: * whenever we select a particular pair of EquivalenceMembers to join, - * we check to see if the pair matches any original clause (in ec_sources) - * or previously-built clause (in ec_derives). This saves memory and allows - * re-use of information cached in RestrictInfos. We also avoid generating - * commutative duplicates, i.e. if the algorithm selects "a.x = b.y" but - * we already have "b.y = a.x", we return the existing clause. + * we check to see if the pair matches any original clause in root's + * eq_sources or previously-built clause (in root's eq_derives). This saves + * memory and allows re-use of information cached in RestrictInfos. We also + * avoid generating commutative duplicates, i.e. if the algorithm selects + * "a.x = b.y" but we already have "b.y = a.x", we return the existing clause. * * If we are considering an outer join, sjinfo is the associated OJ info, * otherwise it can be NULL. @@ -1871,12 +1930,16 @@ generate_join_implied_equalities_broken(PlannerInfo *root, Relids nominal_inner_relids, RelOptInfo *inner_rel) { + Bitmapset *matching_es; List *result = NIL; - ListCell *lc; + int i; - foreach(lc, ec->ec_sources) + matching_es = get_ec_source_indexes(root, ec, nominal_join_relids); + i = -1; + while ((i = bms_next_member(matching_es, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); Relids clause_relids = restrictinfo->required_relids; if (bms_is_subset(clause_relids, nominal_join_relids) && @@ -1888,12 +1951,12 @@ generate_join_implied_equalities_broken(PlannerInfo *root, /* * If we have to translate, just brute-force apply adjust_appendrel_attrs * to all the RestrictInfos at once. This will result in returning - * RestrictInfos that are not listed in ec_derives, but there shouldn't be + * RestrictInfos that are not listed in eq_derives, but there shouldn't be * any duplication, and it's a sufficiently narrow corner case that we * shouldn't sweat too much over it anyway. * * Since inner_rel might be an indirect descendant of the baserel - * mentioned in the ec_sources clauses, we have to be prepared to apply + * mentioned in the eq_sources clauses, we have to be prepared to apply * multiple levels of Var translation. */ if (IS_OTHER_REL(inner_rel) && result != NIL) @@ -1955,10 +2018,11 @@ create_join_clause(PlannerInfo *root, EquivalenceMember *rightem, EquivalenceClass *parent_ec) { + Bitmapset *matches; RestrictInfo *rinfo; RestrictInfo *parent_rinfo = NULL; - ListCell *lc; MemoryContext oldcontext; + int i; /* * Search to see if we already built a RestrictInfo for this pair of @@ -1969,9 +2033,12 @@ create_join_clause(PlannerInfo *root, * it's not identical, it'd better have the same effects, or the operator * families we're using are broken. */ - foreach(lc, ec->ec_sources) + matches = bms_int_members(get_ec_source_indexes_strict(root, ec, leftem->em_relids), + get_ec_source_indexes_strict(root, ec, rightem->em_relids)); + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -1982,9 +2049,13 @@ create_join_clause(PlannerInfo *root, return rinfo; } - foreach(lc, ec->ec_derives) + matches = bms_int_members(get_ec_derive_indexes_strict(root, ec, leftem->em_relids), + get_ec_derive_indexes_strict(root, ec, rightem->em_relids)); + + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -2042,7 +2113,7 @@ create_join_clause(PlannerInfo *root, rinfo->left_em = leftem; rinfo->right_em = rightem; /* and save it for possible re-use */ - ec->ec_derives = lappend(ec->ec_derives, rinfo); + add_eq_derive(root, ec, rinfo); MemoryContextSwitchTo(oldcontext); @@ -2721,16 +2792,19 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, * Returns NULL if no such clause can be found. */ RestrictInfo * -find_derived_clause_for_ec_member(EquivalenceClass *ec, +find_derived_clause_for_ec_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em) { - ListCell *lc; + int i; Assert(ec->ec_has_const); Assert(!em->em_is_const); - foreach(lc, ec->ec_derives) + + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); /* * generate_base_implied_equalities_const will have put non-const @@ -3335,7 +3409,7 @@ have_relevant_eclass_joinclause(PlannerInfo *root, * surely be true if both of them overlap ec_relids.) * * Note we don't test ec_broken; if we did, we'd need a separate code - * path to look through ec_sources. Checking the membership anyway is + * path to look through eq_sources. Checking the membership anyway is * OK as a possibly-overoptimistic heuristic. * * We don't test ec_has_const either, even though a const eclass won't @@ -3423,7 +3497,7 @@ eclass_useful_for_merging(PlannerInfo *root, /* * Note we don't test ec_broken; if we did, we'd need a separate code path - * to look through ec_sources. Checking the members anyway is OK as a + * to look through eq_sources. Checking the members anyway is OK as a * possibly-overoptimistic heuristic. */ @@ -3576,3 +3650,179 @@ get_common_eclass_indexes(PlannerInfo *root, Relids relids1, Relids relids2) /* Calculate and return the common EC indexes, recycling the left input. */ return bms_int_members(rel1ecs, rel2ecs); } + +/* + * get_ec_source_indexes + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_esis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_esis, ec->ec_source_indexes); +} + +/* + * get_ec_source_indexes_strict + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *esis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + esis = bms_intersect(ec->ec_source_indexes, + rte->eclass_source_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + esis = bms_int_members(esis, rte->eclass_source_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return esis; +} + +/* + * get_ec_derive_indexes + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + * + * XXX is this function even needed? + */ +Bitmapset * +get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_edis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_edis, ec->ec_derive_indexes); +} + +/* + * get_ec_derive_indexes_strict + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *edis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + edis = bms_intersect(ec->ec_derive_indexes, + rte->eclass_derive_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + edis = bms_int_members(edis, rte->eclass_derive_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return edis; +} diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index f0735f3c41..38328d945a 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -669,6 +669,7 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid) { ListCell *lc; + int i; /* Fix up the EC's overall relids */ ec->ec_relids = bms_del_member(ec->ec_relids, relid); @@ -702,9 +703,10 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, } /* Fix up the source clauses, in case we can re-use them later */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); remove_rel_from_restrictinfo(rinfo, relid, ojrelid); } @@ -714,7 +716,7 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, * drop them. (At this point, any such clauses would be base restriction * clauses, which we'd not need anymore anyway.) */ - ec->ec_derives = NIL; + ec->ec_derive_indexes = NULL; } /* diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 667723b675..9420259e4b 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -646,6 +646,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, root->multiexpr_params = NIL; root->join_domains = NIL; root->eq_classes = NIL; + root->eq_sources = NIL; + root->eq_derives = NIL; root->ec_merging_done = false; root->last_rinfo_serial = 0; root->all_result_relids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index aa83dd3636..9fc0b69580 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -993,6 +993,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, subroot->multiexpr_params = NIL; subroot->join_domains = NIL; subroot->eq_classes = NIL; + subroot->eq_sources = NIL; + subroot->eq_derives = NIL; subroot->ec_merging_done = false; subroot->last_rinfo_serial = 0; subroot->all_result_relids = NULL; diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3501fbbeed..a65a76ace9 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -483,6 +483,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); + /* + * We do not want to inherit the EquivalenceMember indexes of the parent + * to its child + */ + childrte->eclass_source_indexes = NULL; + childrte->eclass_derive_indexes = NULL; + Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index b3181f34ae..d365bb5c72 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1194,6 +1194,12 @@ typedef struct RangeTblEntry bool inh; /* inheritance requested? */ bool inFromCl; /* present in FROM clause? */ List *securityQuals; /* security barrier quals to apply, if any */ + Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources + * list for RestrictInfos that mention + * this relation */ + Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives + * list for RestrictInfos that mention + * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 8c55a2bcf0..1c5d7bc0c1 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -318,6 +318,12 @@ struct PlannerInfo /* list of active EquivalenceClasses */ List *eq_classes; + /* list of source RestrictInfos used to build EquivalenceClasses */ + List *eq_sources; + + /* list of RestrictInfos derived from EquivalenceClasses */ + List *eq_derives; + /* set true once ECs are canonical */ bool ec_merging_done; @@ -1369,6 +1375,41 @@ typedef struct JoinDomain * entry: consider SELECT random() AS a, random() AS b ... ORDER BY b,a. * So we record the SortGroupRef of the originating sort clause. * + * TODO: We should update the following comments. + * + * At various locations in the query planner, we must search for + * EquivalenceMembers within a given EquivalenceClass. For the common case, + * an EquivalenceClass does not have a large number of EquivalenceMembers, + * however, in cases such as planning queries to partitioned tables, the number + * of members can become large. To maintain planning performance, we make use + * of a bitmap index to allow us to quickly find EquivalenceMembers in a given + * EquivalenceClass belonging to a given relation or set of relations. This + * is done by storing a list of EquivalenceMembers belonging to all + * EquivalenceClasses in PlannerInfo and storing a Bitmapset for each + * RelOptInfo which has a bit set for each EquivalenceMember in that list + * which relates to the given relation. We also store a Bitmapset to mark all + * of the indexes in the PlannerInfo's list of EquivalenceMembers in the + * EquivalenceClass. We can quickly find the interesting indexes into the + * PlannerInfo's list by performing a bit-wise AND on the RelOptInfo's + * Bitmapset and the EquivalenceClasses. + * + * To further speed up the lookup of EquivalenceMembers we also record the + * non-child indexes. This allows us to deal with fewer bits for searches + * that don't require "is_child" EquivalenceMembers. We must also store the + * indexes into EquivalenceMembers which have no relids mentioned as some + * searches require that. + * + * Additionally, we also store the EquivalenceMembers of a given + * EquivalenceClass in the ec_members list. Technically, we could obtain this + * from looking at the bits in ec_member_indexes, however, finding all members + * of a given EquivalenceClass is common enough that it pays to have fast + * access to a dense list containing all members. + * + * The source and derived RestrictInfos are indexed in a similar method, + * although we don't maintain a List in the EquivalenceClass for these. These + * must be looked up in PlannerInfo using the ec_source_indexes and + * ec_derive_indexes Bitmapsets. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1391,8 +1432,10 @@ typedef struct EquivalenceClass * em_relids is empty */ List *ec_rel_members; /* list of EquivalenceMembers whose * em_relids is not empty */ - List *ec_sources; /* list of generating RestrictInfos */ - List *ec_derives; /* list of derived RestrictInfos */ + Bitmapset *ec_source_indexes; /* indexes into PlannerInfo's eq_sources + * list of generating RestrictInfos */ + Bitmapset *ec_derive_indexes; /* indexes into PlannerInfo's eq_derives + * list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except * for child members (see below) */ bool ec_has_const; /* any pseudoconstants in ec_members? */ diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9bc0c36b93..d071f6d0fc 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -163,7 +163,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2); extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); -extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, +extern RestrictInfo *find_derived_clause_for_ec_member(PlannerInfo *root, + EquivalenceClass *ec, EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, @@ -195,6 +196,18 @@ extern bool eclass_useful_for_merging(PlannerInfo *root, extern bool is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist); extern bool is_redundant_with_indexclauses(RestrictInfo *rinfo, List *indexclauses); +extern Bitmapset *get_ec_source_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_source_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); /* * pathkeys.c -- 2.42.0.windows.2 [application/octet-stream] v23-0003-Solve-conflict-with-self-join-removal.patch (3.8K, ../../CAJ2pMkaaY4FgGSO2z9X_XOoqtLMUhyx6cPZFQbxmsjLgLpEd2Q@mail.gmail.com/5-v23-0003-Solve-conflict-with-self-join-removal.patch) download | inline diff: From c7525241e4d3c1adac3328420a61ada5aa799107 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Mon, 11 Dec 2023 12:19:55 +0900 Subject: [PATCH v23 3/6] Solve conflict with self join removal Written by Alena Rybakina <[email protected]> [1] with modifications by me. [1] https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/optimizer/plan/analyzejoins.c | 49 ++++++++++++++++------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 38328d945a..49f3c17057 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -1589,10 +1589,12 @@ replace_relid(Relids relids, int oldId, int newId) * delete them. */ static void -update_eclasses(EquivalenceClass *ec, int from, int to) +update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) { List *new_members = NIL; - List *new_sources = NIL; + Bitmapset *new_source_indexes = NULL; + int i; + int j; ListCell *lc; ListCell *lc1; @@ -1634,18 +1636,28 @@ update_eclasses(EquivalenceClass *ec, int from, int to) list_free(ec->ec_members); ec->ec_members = new_members; - list_free(ec->ec_derives); - ec->ec_derives = NULL; + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) + { + /* + * Can't delete the element because we would need to rebuild all + * the eq_derives indexes. But set a nuke to detect potential problems. + */ + list_nth_cell(root->eq_derives, i)->ptr_value = NULL; + } + bms_free(ec->ec_derive_indexes); + ec->ec_derive_indexes = NULL; /* Update EC source expressions */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); bool is_redundant = false; if (!bms_is_member(from, rinfo->required_relids)) { - new_sources = lappend(new_sources, rinfo); + new_source_indexes = bms_add_member(new_source_indexes, i); continue; } @@ -1656,9 +1668,10 @@ update_eclasses(EquivalenceClass *ec, int from, int to) * redundancy with existing ones. We don't have to check for * redundancy with derived clauses, because we've just deleted them. */ - foreach(lc1, new_sources) + j = -1; + while ((j = bms_next_member(new_source_indexes, j)) >= 0) { - RestrictInfo *other = lfirst_node(RestrictInfo, lc1); + RestrictInfo *other = list_nth_node(RestrictInfo, root->eq_sources, j); if (!equal(rinfo->clause_relids, other->clause_relids)) continue; @@ -1670,12 +1683,20 @@ update_eclasses(EquivalenceClass *ec, int from, int to) } } - if (!is_redundant) - new_sources = lappend(new_sources, rinfo); + if (is_redundant) + { + /* + * Can't delete the element because we would need to rebuild all + * the eq_sources indexes. But set a nuke to detect potential problems. + */ + list_nth_cell(root->eq_sources, i)->ptr_value = NULL; + } + else + new_source_indexes = bms_add_member(new_source_indexes, i); } - list_free(ec->ec_sources); - ec->ec_sources = new_sources; + bms_free(ec->ec_source_indexes); + ec->ec_source_indexes = new_source_indexes; ec->ec_relids = replace_relid(ec->ec_relids, from, to); } @@ -1855,7 +1876,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, { EquivalenceClass *ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - update_eclasses(ec, toRemove->relid, toKeep->relid); + update_eclasses(root, ec, toRemove->relid, toKeep->relid); toKeep->eclass_indexes = bms_add_member(toKeep->eclass_indexes, i); } -- 2.42.0.windows.2 [application/octet-stream] v23-0004-Fix-a-bug-related-to-atomic-function.patch (3.0K, ../../CAJ2pMkaaY4FgGSO2z9X_XOoqtLMUhyx6cPZFQbxmsjLgLpEd2Q@mail.gmail.com/6-v23-0004-Fix-a-bug-related-to-atomic-function.patch) download | inline diff: From 914045526469d8010becaa4e3ebebff23167681a Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Mon, 11 Dec 2023 12:20:20 +0900 Subject: [PATCH v23 4/6] Fix a bug related to atomic function Author: Alena Rybakina <[email protected]> https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/nodes/read.c | 20 ++++++++++++++++++++ src/backend/nodes/readfuncs.c | 24 ++++++++++++++++++++++-- src/include/nodes/readfuncs.h | 2 ++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/backend/nodes/read.c b/src/backend/nodes/read.c index 969d0ec199..39c5a39fa7 100644 --- a/src/backend/nodes/read.c +++ b/src/backend/nodes/read.c @@ -514,3 +514,23 @@ nodeRead(const char *token, int tok_len) return (void *) result; } + +/* + * pg_strtok_save_context - + * Save context initialized by stringToNode. + */ +void +pg_strtok_save_context(const char **pcontext) +{ + *pcontext = pg_strtok_ptr; +} + +/* + * pg_strtok_restore_context - + * Resore saved context. + */ +void +pg_strtok_restore_context(const char *context) +{ + pg_strtok_ptr = context; +} diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index deebfaf994..0b713838dd 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -191,6 +191,26 @@ nullable_string(const char *token, int length) return debackslash(token, length); } +/* Read an equivalence field (anything written as ":fldname %u") and check it */ +#define READ_EQ_BITMAPSET_FIELD_CHECK(fldname) \ +{ \ + int save_length = length; \ + const char *context; \ + pg_strtok_save_context(&context); \ + token = pg_strtok(&length); \ + if (length > 0 && strncmp(token, ":"#fldname, strlen(":"#fldname))) \ + { \ + /* "fldname" field was not found - fill it and restore context. */ \ + local_node->fldname = NULL; \ + pg_strtok_restore_context(context); \ + length = save_length; \ + } \ + else \ + { \ + local_node->fldname = _readBitmapset(); \ + } \ +} + /* * _readBitmapset @@ -574,8 +594,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_BITMAPSET_FIELD(eclass_source_indexes); - READ_BITMAPSET_FIELD(eclass_derive_indexes); + READ_EQ_BITMAPSET_FIELD_CHECK(eclass_source_indexes); + READ_EQ_BITMAPSET_FIELD_CHECK(eclass_derive_indexes); READ_DONE(); } diff --git a/src/include/nodes/readfuncs.h b/src/include/nodes/readfuncs.h index 8466038ed0..0dc2ed5112 100644 --- a/src/include/nodes/readfuncs.h +++ b/src/include/nodes/readfuncs.h @@ -29,6 +29,8 @@ extern PGDLLIMPORT bool restore_location_fields; extern const char *pg_strtok(int *length); extern char *debackslash(const char *token, int length); extern void *nodeRead(const char *token, int tok_len); +extern void pg_strtok_save_context(const char **pcontext); +extern void pg_strtok_restore_context(const char *context); /* * prototypes for functions in readfuncs.c -- 2.42.0.windows.2 [application/octet-stream] v23-0005-Remove-all-references-between-RestrictInfo-and-i.patch (2.6K, ../../CAJ2pMkaaY4FgGSO2z9X_XOoqtLMUhyx6cPZFQbxmsjLgLpEd2Q@mail.gmail.com/7-v23-0005-Remove-all-references-between-RestrictInfo-and-i.patch) download | inline diff: From 0c61c44e693e12ca918bdd24ab266423b0758395 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Thu, 11 Jan 2024 16:18:36 +0900 Subject: [PATCH v23 5/6] Remove all references between RestrictInfo and its relating RangeTblEntry This commit adds code to adjust eclass_derive_indexes and eclass_source_indexes during self-join elimination. Note that this commit fails some regression tests. The failures will be fixed in the next commit. See its commit message for more details. --- src/backend/optimizer/plan/analyzejoins.c | 34 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 49f3c17057..4081827e3a 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -1639,9 +1639,25 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) i = -1; while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); + + /* + * Remove all references between this RestrictInfo and its relating + * RangeTblEntry. + */ + j = -1; + while ((j = bms_next_member(rinfo->clause_relids, j)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[j]; + + Assert(bms_is_member(i, rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, i); + } + /* * Can't delete the element because we would need to rebuild all - * the eq_derives indexes. But set a nuke to detect potential problems. + * the eq_derives indexes. But set a null to detect potential problems. */ list_nth_cell(root->eq_derives, i)->ptr_value = NULL; } @@ -1685,9 +1701,23 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) if (is_redundant) { + /* + * Remove all references between this RestrictInfo and its relating + * RangeTblEntry. + */ + j = -1; + while ((j = bms_next_member(rinfo->clause_relids, j)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[j]; + + Assert(bms_is_member(i, rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, i); + } + /* * Can't delete the element because we would need to rebuild all - * the eq_sources indexes. But set a nuke to detect potential problems. + * the eq_sources indexes. But set a null to detect potential problems. */ list_nth_cell(root->eq_sources, i)->ptr_value = NULL; } -- 2.42.0.windows.2 [application/octet-stream] v23-0006-Introduce-update_clause_relids-for-updating-Rest.patch (19.0K, ../../CAJ2pMkaaY4FgGSO2z9X_XOoqtLMUhyx6cPZFQbxmsjLgLpEd2Q@mail.gmail.com/8-v23-0006-Introduce-update_clause_relids-for-updating-Rest.patch) download | inline diff: From 6bcce92df6dc597e050b7161e5555f3bfd7da841 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Thu, 11 Jan 2024 16:18:38 +0900 Subject: [PATCH v23 6/6] Introduce update_clause_relids() for updating RestrictInfo->clause_relids Originally, the update for RestrictInfo->clause_relids was done by directly setting the field. However, this failed to update the corresponding indexes, such as eclass_source_indexes and eclass_derive_indexes. This commit provides a helper function to update the field with fixing the indexes. --- src/backend/optimizer/path/equivclass.c | 120 ++++++++++++++++++++++ src/backend/optimizer/plan/analyzejoins.c | 81 +++++++++------ src/backend/optimizer/util/appendinfo.c | 2 + src/backend/optimizer/util/restrictinfo.c | 5 + src/include/nodes/pathnodes.h | 11 +- src/include/optimizer/paths.h | 2 + 6 files changed, 191 insertions(+), 30 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index d1019315ea..3f4df1a8c7 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -623,6 +623,8 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); root->eq_sources = lappend(root->eq_sources, rinfo); + Assert(rinfo->eq_sources_index == -1); + rinfo->eq_sources_index = source_idx; i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) @@ -645,6 +647,8 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); root->eq_derives = lappend(root->eq_derives, rinfo); + Assert(rinfo->eq_derives_index == -1); + rinfo->eq_derives_index = derive_idx; i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) @@ -656,6 +660,122 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) } } +/* + * update_clause_relids + * Update RestrictInfo->clause_relids with adjusting the relevant indexes. + * The clause_relids field must be updated through this function, not by + * setting the field directly. + */ +void +update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids) +{ + int i; + Relids removing_relids; + Relids adding_relids; +#ifdef USE_ASSERT_CHECKING + Relids common_relids; +#endif + + /* + * If there is nothing to do, we return immediately after setting the + * clause_relids field. + */ + if (rinfo->eq_sources_index == -1 && rinfo->eq_derives_index == -1) + { + rinfo->clause_relids = new_clause_relids; + return; + } + + /* + * Remove any references between this RestrictInfo and RangeTblEntries + * that are no longer relevant. + */ + removing_relids = bms_difference(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(removing_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(removing_relids); + + /* + * Add references between this RestrictInfo and RangeTblEntries that will + * be relevant. + */ + adding_relids = bms_difference(new_clause_relids, rinfo->clause_relids); + i = -1; + while ((i = bms_next_member(adding_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(!bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_add_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(!bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_add_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(adding_relids); + +#ifdef USE_ASSERT_CHECKING + /* + * Verify that all indexes are set for common Relids. + */ + common_relids = bms_intersect(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(common_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + } + } + bms_free(common_relids); +#endif + + /* + * Since we have done everything to adjust the indexes, we can set the + * clause_relids field. + */ + rinfo->clause_relids = new_clause_relids; +} + /* * get_eclass_for_sort_expr diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 4081827e3a..2da2a6c14e 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -39,6 +39,7 @@ */ typedef struct { + PlannerInfo *root; int from; int to; } ReplaceVarnoContext; @@ -59,7 +60,8 @@ static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); static void remove_leftjoinrel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); -static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, +static void remove_rel_from_restrictinfo(PlannerInfo *root, + RestrictInfo *rinfo, int relid, int ojrelid); static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); @@ -76,7 +78,7 @@ static bool is_innerrel_unique_for(PlannerInfo *root, List *restrictlist, List **extra_clauses); static Bitmapset *replace_relid(Relids relids, int oldId, int newId); -static void replace_varno(Node *node, int from, int to); +static void replace_varno(PlannerInfo *root, Node *node, int from, int to); static bool replace_varno_walker(Node *node, ReplaceVarnoContext *ctx); static int self_join_candidates_cmp(const void *a, const void *b); @@ -395,7 +397,7 @@ remove_rel_from_query(PlannerInfo *root, RelOptInfo *rel, } /* Update lateral references. */ - replace_varno((Node *) otherrel->lateral_vars, relid, subst); + replace_varno(root, (Node *) otherrel->lateral_vars, relid, subst); } /* @@ -432,7 +434,7 @@ remove_rel_from_query(PlannerInfo *root, RelOptInfo *rel, sjinf->commute_below_l = replace_relid(sjinf->commute_below_l, ojrelid, subst); sjinf->commute_below_r = replace_relid(sjinf->commute_below_r, ojrelid, subst); - replace_varno((Node *) sjinf->semi_rhs_exprs, relid, subst); + replace_varno(root, (Node *) sjinf->semi_rhs_exprs, relid, subst); } /* @@ -477,7 +479,7 @@ remove_rel_from_query(PlannerInfo *root, RelOptInfo *rel, phv->phrels = replace_relid(phv->phrels, relid, subst); phv->phrels = replace_relid(phv->phrels, ojrelid, subst); Assert(!bms_is_empty(phv->phrels)); - replace_varno((Node *) phv->phexpr, relid, subst); + replace_varno(root, (Node *) phv->phexpr, relid, subst); Assert(phv->phnullingrels == NULL); /* no need to adjust */ } } @@ -551,7 +553,7 @@ remove_leftjoinrel_from_query(PlannerInfo *root, int relid, * that any such PHV is safe (and updated its ph_eval_at), so we * can just drop those references. */ - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); /* * Cross-check that the clause itself does not reference the @@ -608,16 +610,24 @@ remove_leftjoinrel_from_query(PlannerInfo *root, int relid, * we have to also clean up the sub-clauses. */ static void -remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) +remove_rel_from_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo, int relid, + int ojrelid) { + Relids new_clause_relids; + + /* + * The 'new_clause_relids' passed to update_clause_relids() must be a + * different instance from the current rinfo->clause_relids, so we make a + * copy of them. + */ + new_clause_relids = bms_copy(rinfo->clause_relids); + new_clause_relids = bms_del_member(new_clause_relids, relid); + new_clause_relids = bms_del_member(new_clause_relids, ojrelid); + update_clause_relids(root, rinfo, new_clause_relids); /* - * The clause_relids probably aren't shared with anything else, but let's + * The required_relids probably aren't shared with anything else, but let's * copy them just to be sure. */ - rinfo->clause_relids = bms_copy(rinfo->clause_relids); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, relid); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, ojrelid); - /* Likewise for required_relids */ rinfo->required_relids = bms_copy(rinfo->required_relids); rinfo->required_relids = bms_del_member(rinfo->required_relids, relid); rinfo->required_relids = bms_del_member(rinfo->required_relids, ojrelid); @@ -642,14 +652,14 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) { RestrictInfo *rinfo2 = lfirst_node(RestrictInfo, lc2); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } else { RestrictInfo *rinfo2 = castNode(RestrictInfo, orarg); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } } @@ -708,7 +718,7 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, { RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); } /* @@ -1448,13 +1458,14 @@ is_innerrel_unique_for(PlannerInfo *root, * Replace each occurrence of removing relid with the keeping one */ static void -replace_varno(Node *node, int from, int to) +replace_varno(PlannerInfo *root, Node *node, int from, int to) { ReplaceVarnoContext ctx; if (to <= 0) return; + ctx.root = root; ctx.from = from; ctx.to = to; replace_varno_walker(node, &ctx); @@ -1498,9 +1509,9 @@ replace_varno_walker(Node *node, ReplaceVarnoContext *ctx) if (bms_is_member(ctx->from, rinfo->clause_relids)) { - replace_varno((Node *) rinfo->clause, ctx->from, ctx->to); - replace_varno((Node *) rinfo->orclause, ctx->from, ctx->to); - rinfo->clause_relids = replace_relid(rinfo->clause_relids, ctx->from, ctx->to); + replace_varno(ctx->root, (Node *) rinfo->clause, ctx->from, ctx->to); + replace_varno(ctx->root, (Node *) rinfo->orclause, ctx->from, ctx->to); + update_clause_relids(ctx->root, rinfo, replace_relid(rinfo->clause_relids, ctx->from, ctx->to)); rinfo->left_relids = replace_relid(rinfo->left_relids, ctx->from, ctx->to); rinfo->right_relids = replace_relid(rinfo->right_relids, ctx->from, ctx->to); } @@ -1613,7 +1624,7 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) em->em_jdomain->jd_relids = replace_relid(em->em_jdomain->jd_relids, from, to); /* We only process inner joins */ - replace_varno((Node *) em->em_expr, from, to); + replace_varno(root, (Node *) em->em_expr, from, to); foreach(lc1, new_members) { @@ -1660,6 +1671,12 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) * the eq_derives indexes. But set a null to detect potential problems. */ list_nth_cell(root->eq_derives, i)->ptr_value = NULL; + + /* + * Since this RestrictInfo no longer exists in root->eq_derives, we + * must reset the stored index. + */ + rinfo->eq_derives_index = -1; } bms_free(ec->ec_derive_indexes); ec->ec_derive_indexes = NULL; @@ -1677,7 +1694,7 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) continue; } - replace_varno((Node *) rinfo, from, to); + replace_varno(root, (Node *) rinfo, from, to); /* * After switching the clause to the remaining relation, check it for @@ -1720,6 +1737,12 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) * the eq_sources indexes. But set a null to detect potential problems. */ list_nth_cell(root->eq_sources, i)->ptr_value = NULL; + + /* + * Since this RestrictInfo no longer exists in root->eq_sources, we + * must reset the stored index. + */ + rinfo->eq_sources_index = -1; } else new_source_indexes = bms_add_member(new_source_indexes, i); @@ -1782,7 +1805,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, int i; List *jinfo_candidates = NIL; List *binfo_candidates = NIL; - ReplaceVarnoContext ctx = {.from = toRemove->relid,.to = toKeep->relid}; + ReplaceVarnoContext ctx = {.root = root,.from = toRemove->relid,.to = toKeep->relid}; Assert(toKeep->relid != -1); @@ -1798,7 +1821,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); remove_join_clause_from_rels(root, rinfo, rinfo->required_relids); - replace_varno((Node *) rinfo, toRemove->relid, toKeep->relid); + replace_varno(root, (Node *) rinfo, toRemove->relid, toKeep->relid); if (bms_membership(rinfo->required_relids) == BMS_MULTIPLE) jinfo_candidates = lappend(jinfo_candidates, rinfo); @@ -1818,7 +1841,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); - replace_varno((Node *) rinfo, toRemove->relid, toKeep->relid); + replace_varno(root, (Node *) rinfo, toRemove->relid, toKeep->relid); if (bms_membership(rinfo->required_relids) == BMS_MULTIPLE) jinfo_candidates = lappend(jinfo_candidates, rinfo); @@ -1918,7 +1941,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, { Node *node = lfirst(lc); - replace_varno(node, toRemove->relid, toKeep->relid); + replace_varno(root, node, toRemove->relid, toKeep->relid); if (!list_member(toKeep->reltarget->exprs, node)) toKeep->reltarget->exprs = lappend(toKeep->reltarget->exprs, node); } @@ -1970,9 +1993,9 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, remove_rel_from_query(root, toRemove, toKeep->relid, NULL, NULL); /* At last, replace varno in root targetlist and HAVING clause */ - replace_varno((Node *) root->processed_tlist, + replace_varno(root, (Node *) root->processed_tlist, toRemove->relid, toKeep->relid); - replace_varno((Node *) root->processed_groupClause, + replace_varno(root, (Node *) root->processed_groupClause, toRemove->relid, toKeep->relid); replace_relid(root->all_result_relids, toRemove->relid, toKeep->relid); replace_relid(root->leaf_result_relids, toRemove->relid, toKeep->relid); @@ -2049,7 +2072,7 @@ split_selfjoin_quals(PlannerInfo *root, List *joinquals, List **selfjoinquals, * when we have cast of the same var to different (but compatible) * types. */ - replace_varno(rightexpr, bms_singleton_member(rinfo->right_relids), + replace_varno(root, rightexpr, bms_singleton_member(rinfo->right_relids), bms_singleton_member(rinfo->left_relids)); if (equal(leftexpr, rightexpr)) @@ -2090,7 +2113,7 @@ match_unique_clauses(PlannerInfo *root, RelOptInfo *outer, List *uclauses, bms_is_empty(rinfo->right_relids)); clause = (Expr *) copyObject(rinfo->clause); - replace_varno((Node *) clause, relid, outer->relid); + replace_varno(root, (Node *) clause, relid, outer->relid); iclause = bms_is_empty(rinfo->left_relids) ? get_rightop(clause) : get_leftop(clause); diff --git a/src/backend/optimizer/util/appendinfo.c b/src/backend/optimizer/util/appendinfo.c index 51fdeace7d..27e4b093d7 100644 --- a/src/backend/optimizer/util/appendinfo.c +++ b/src/backend/optimizer/util/appendinfo.c @@ -491,6 +491,8 @@ adjust_appendrel_attrs_mutator(Node *node, newinfo->right_bucketsize = -1; newinfo->left_mcvfreq = -1; newinfo->right_mcvfreq = -1; + newinfo->eq_sources_index = -1; + newinfo->eq_derives_index = -1; return (Node *) newinfo; } diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 0b406e9334..59ba503ecb 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -247,6 +247,9 @@ make_restrictinfo_internal(PlannerInfo *root, restrictinfo->left_hasheqoperator = InvalidOid; restrictinfo->right_hasheqoperator = InvalidOid; + restrictinfo->eq_sources_index = -1; + restrictinfo->eq_derives_index = -1; + return restrictinfo; } @@ -403,6 +406,8 @@ commute_restrictinfo(RestrictInfo *rinfo, Oid comm_op) result->right_mcvfreq = rinfo->left_mcvfreq; result->left_hasheqoperator = InvalidOid; result->right_hasheqoperator = InvalidOid; + result->eq_sources_index = -1; + result->eq_derives_index = -1; return result; } diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 1c5d7bc0c1..b856722533 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -2622,7 +2622,12 @@ typedef struct RestrictInfo /* number of base rels in clause_relids */ int num_base_rels pg_node_attr(equal_ignore); - /* The relids (varnos+varnullingrels) actually referenced in the clause: */ + /* + * The relids (varnos+varnullingrels) actually referenced in the clause. + * + * NOTE: This field must be updated through update_clause_relids(), not by + * setting the field directly. + */ Relids clause_relids pg_node_attr(equal_ignore); /* The set of relids required to evaluate the clause: */ @@ -2737,6 +2742,10 @@ typedef struct RestrictInfo /* hash equality operators used for memoize nodes, else InvalidOid */ Oid left_hasheqoperator pg_node_attr(equal_ignore); Oid right_hasheqoperator pg_node_attr(equal_ignore); + + /* the index within root->eq_sources and root->eq_derives */ + int eq_sources_index pg_node_attr(equal_ignore); + int eq_derives_index pg_node_attr(equal_ignore); } RestrictInfo; /* diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index d071f6d0fc..c23b177f65 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -128,6 +128,8 @@ extern bool process_equivalence(PlannerInfo *root, extern Expr *canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation); extern void reconsider_outer_join_clauses(PlannerInfo *root); +extern void update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids); extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Expr *expr, List *opfamilies, -- 2.42.0.windows.2 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-02-12 21:19 Alena Rybakina <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Alena Rybakina @ 2024-02-12 21:19 UTC (permalink / raw) To: Yuya Watari <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; Ashutosh Bapat <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hi! Sorry my delayed reply too. On 17.01.2024 12:33, Yuya Watari wrote: > Hello Alena, > > Thank you for your quick response, and I'm sorry for my delayed reply. > > On Sun, Dec 17, 2023 at 12:41 AM Alena Rybakina > <[email protected]> wrote: >> I thought about this earlier and was worried that the index links of the equivalence classes might not be referenced correctly for outer joins, >> so I decided to just overwrite them and reset the previous ones. > Thank you for pointing this out. I have investigated this problem and > found a potential bug place. The code quoted below modifies > RestrictInfo's clause_relids. Here, our indexes, namely > eclass_source_indexes and eclass_derive_indexes, are based on > clause_relids, so they should be adjusted after the modification. > However, my patch didn't do that, so it may have missed some > references. The same problem occurs in places other than the quoted > one. > > ===== > /* > * Walker function for replace_varno() > */ > static bool > replace_varno_walker(Node *node, ReplaceVarnoContext *ctx) > { > ... > else if (IsA(node, RestrictInfo)) > { > RestrictInfo *rinfo = (RestrictInfo *) node; > ... > > if (bms_is_member(ctx->from, rinfo->clause_relids)) > { > replace_varno((Node *) rinfo->clause, ctx->from, ctx->to); > replace_varno((Node *) rinfo->orclause, ctx->from, ctx->to); > rinfo->clause_relids = replace_relid(rinfo->clause_relids, > ctx->from, ctx->to); > ... > } > ... > } > ... > } > ===== > > I have attached a new version of the patch, v23, to fix this problem. > v23-0006 adds a helper function called update_clause_relids(). This > function modifies RestrictInfo->clause_relids while adjusting its > related indexes. I have also attached a sanity check patch > (sanity-check.txt) to this email. This sanity check patch verifies > that there are no missing references between RestrictInfos and our > indexes. I don't intend to commit this patch, but it helps find > potential bugs. v23 passes this sanity check, but the v21 you > submitted before does not. This means that the adjustment by > update_clause_relids() is needed to prevent missing references after > modifying clause_relids. I'd appreciate your letting me know if v23 > doesn't solve your concern. > > One of the things I don't think is good about my approach is that it > adds some complexity to the code. In my approach, all modifications to > clause_relids must be done through the update_clause_relids() > function, but enforcing this rule is not so easy. In this sense, my > patch may need to be simplified more. > >> this is due to the fact that I explained before: we zeroed the values indicated by the indexes, >> then this check is not correct either - since the zeroed value indicated by the index is correct. >> That's why I removed this check. > Thank you for letting me know. I fixed this in v23-0005 to adjust the > indexes in update_eclasses(). With this change, the assertion check > will be correct. > Yes, it is working correctly now with the assertion check. I suppose it's better to add this code with an additional comment and a recommendation for other developers to use it for checking in case of manipulations with the list of equivalences. -- Regards, Alena Rybakina Postgres Professional: http://www.postgrespro.com The Russian Postgres Company ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-02-28 11:18 Yuya Watari <[email protected]> parent: Alena Rybakina <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-02-28 11:18 UTC (permalink / raw) To: Alena Rybakina <[email protected]>; +Cc: Andrei Lepikhov <[email protected]>; Ashutosh Bapat <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hello, On Tue, Feb 13, 2024 at 6:19 AM Alena Rybakina <[email protected]> wrote: > > Yes, it is working correctly now with the assertion check. I suppose > it's better to add this code with an additional comment and a > recommendation for other developers > to use it for checking in case of manipulations with the list of > equivalences. Thank you for your reply and advice. I have added this assertion so that other developers can use it in the future. I also merged recent changes and attached a new version, v24. Since this thread is getting long, I will summarize the patches. 1. v24-0001 This patch is one of the main parts of my optimization. Traditionally, EquivalenceClass has both parent and child members. However, this leads to high iteration costs when there are many child partitions. In v24-0001, EquivalenceClasses no longer have child members. If we need to iterate over child EquivalenceMembers, we use the EquivalenceChildMemberIterator and access the children through the iterator. For more details, see [1] (please note that there are a few design changes from [1]). 2. v24-0002 This patch was made in the previous work with David. Like EquivalenceClass, there are many RestrictInfos in highly partitioned cases. This patch introduces an indexing mechanism to speed up searches for RestrictInfos. 3. v24-0003 v24-0002 adds its indexes to RangeTblEntry, but this is not a good idea. RelOptInfo is the best place. This problem is a workaround because some RelOptInfos can be NULL, so we cannot store indexes to such RelOptInfos. v24-0003 moves the indexes from RangeTblEntry to PlannerInfo. This is still a workaround, and I think it should be reconsidered. [1] https://www.postgresql.org/message-id/CAJ2pMkZk-Nr%3DyCKrGfGLu35gK-D179QPyxaqtJMUkO86y1NmSA%40mail.g... -- Best regards, Yuya Watari Attachments: [application/octet-stream] v24-0001-Speed-up-searches-for-child-EquivalenceMembers.patch (58.1K, ../../CAJ2pMkYRQxNRt=yjxebWxynNVw0onxDSbi4xv9iW3hnGcPcabg@mail.gmail.com/2-v24-0001-Speed-up-searches-for-child-EquivalenceMembers.patch) download | inline diff: From daad7514b257868dc48f156ca1244974660c13dc Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:43:36 +0900 Subject: [PATCH v24 1/3] Speed up searches for child EquivalenceMembers Traditionally, child EquivalenceMembers were in the EquivalenceClass->ec_members. When we wanted to find some child members matching a request, we had to perform a linear search. This search became heavy when tables had many partitions, leading to much planning time. After this commit, child EquivalenceMembers no longer exist in ec_members. Instead, RelOptInfos have them. This change demonstrates a significant performance improvement in planning time. --- contrib/postgres_fdw/postgres_fdw.c | 22 +- src/backend/optimizer/path/equivclass.c | 389 ++++++++++++++++++---- src/backend/optimizer/path/indxpath.c | 35 +- src/backend/optimizer/path/pathkeys.c | 9 +- src/backend/optimizer/plan/analyzejoins.c | 7 +- src/backend/optimizer/plan/createplan.c | 57 ++-- src/backend/optimizer/util/inherit.c | 14 + src/backend/optimizer/util/relnode.c | 88 +++++ src/include/nodes/pathnodes.h | 55 +++ src/include/optimizer/pathnode.h | 18 + src/include/optimizer/paths.h | 12 +- src/tools/pgindent/typedefs.list | 1 + 12 files changed, 592 insertions(+), 115 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 142dcfc995..36fc39733d 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7773,13 +7773,21 @@ conversion_error_callback(void *arg) EquivalenceMember * find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) { - ListCell *lc; - PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + Relids top_parent_rel_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, rel->relids); /* * Note we require !bms_is_empty, else we'd accept constant @@ -7791,6 +7799,7 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) is_foreign_expr(root, rel, em->em_expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -7844,9 +7853,8 @@ find_em_for_rel_target(PlannerInfo *root, EquivalenceClass *ec, if (em->em_is_const) continue; - /* Ignore child members */ - if (em->em_is_child) - continue; + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* Match if same expression (after stripping relabel) */ em_expr = em->em_expr; diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 4bd60a09c6..1931b01cbc 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -33,10 +33,14 @@ #include "utils/lsyscache.h" +static EquivalenceMember *make_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + EquivalenceMember *parent, + Oid datatype); static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, - EquivalenceMember *parent, Oid datatype); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, @@ -69,6 +73,11 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo); static JoinDomain *find_join_domain(PlannerInfo *root, Relids relids); +static void add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel); static Bitmapset *get_eclass_indexes_for_relids(PlannerInfo *root, Relids relids); static Bitmapset *get_common_eclass_indexes(PlannerInfo *root, Relids relids1, @@ -374,7 +383,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); @@ -391,7 +400,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); @@ -423,9 +432,9 @@ process_equivalence(PlannerInfo *root, ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; em1 = add_eq_member(ec, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); em2 = add_eq_member(ec, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -511,11 +520,14 @@ canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation) } /* - * add_eq_member - build a new EquivalenceMember and add it to an EC + * make_eq_member + * + * Build a new EquivalenceMember without adding it to an EC. Note that child + * EquivalenceMembers should not be added to its parent EquivalenceClass. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) +make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) { EquivalenceMember *em = makeNode(EquivalenceMember); @@ -526,6 +538,8 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, em->em_datatype = datatype; em->em_jdomain = jdomain; em->em_parent = parent; + em->em_child_relids = NULL; + em->em_child_joinrel_relids = NULL; if (bms_is_empty(relids)) { @@ -546,11 +560,24 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, { ec->ec_relids = bms_add_members(ec->ec_relids, relids); } - ec->ec_members = lappend(ec->ec_members, em); return em; } +/* + * add_eq_member - build a new EquivalenceMember and add it to an EC + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + + ec->ec_members = lappend(ec->ec_members, em); + return em; +} + /* * get_eclass_for_sort_expr @@ -599,6 +626,17 @@ get_eclass_for_sort_expr(PlannerInfo *root, EquivalenceMember *newem; ListCell *lc1; MemoryContext oldcontext; + Relids top_parent_rel; + + /* + * First, we translate the given Relids to their top-level parents. This is + * required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get child + * members. We can skip such translations if we now see top-level ones, + * i.e., when top_parent_rel is NULL. See the find_relids_top_parents()'s + * definition for more details. + */ + top_parent_rel = find_relids_top_parents(root, rel); /* * Ensure the expression exposes the correct type and collation. @@ -617,7 +655,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; /* * Never match to a volatile EC, except when we are looking at another @@ -632,16 +671,30 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (!equal(opfamilies, cur_ec->ec_opfamilies)) continue; - foreach(lc2, cur_ec->ec_members) + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel); /* * Ignore child members unless they match the request. */ - if (cur_em->em_is_child && - !bms_equal(cur_em->em_relids, rel)) - continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(!cur_em->em_is_child || bms_equal(cur_em->em_relids, rel)); /* * Match constants only within the same JoinDomain (see @@ -665,6 +718,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, return cur_ec; } } + dispose_eclass_child_member_iterator(&it); } /* No match; does caller want a NULL result? */ @@ -702,7 +756,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, expr_relids = pull_varnos(root, (Node *) expr); newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, NULL, opcintype); + jdomain, opcintype); /* * add_eq_member doesn't check for volatile functions, set-returning @@ -768,19 +822,25 @@ get_eclass_for_sort_expr(PlannerInfo *root, * Child EC members are ignored unless they belong to given 'relids'. */ EquivalenceMember * -find_ec_member_matching_expr(EquivalenceClass *ec, +find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, Expr *expr, Relids relids) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); /* We ignore binary-compatible relabeling on both ends */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); Expr *emexpr; /* @@ -790,6 +850,11 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -807,6 +872,7 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (equal(emexpr, expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -839,11 +905,17 @@ find_computable_ec_member(PlannerInfo *root, Relids relids, bool require_parallel_safe) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); List *exprvars; ListCell *lc2; @@ -854,6 +926,11 @@ find_computable_ec_member(PlannerInfo *root, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -887,6 +964,7 @@ find_computable_ec_member(PlannerInfo *root, return em; /* found usable expression */ } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -950,7 +1028,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, { Expr *targetexpr = (Expr *) lfirst(lc); - em = find_ec_member_matching_expr(ec, targetexpr, rel->relids); + em = find_ec_member_matching_expr(root, ec, targetexpr, rel->relids); if (!em) continue; @@ -1570,7 +1648,12 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *new_members = NIL; List *outer_members = NIL; List *inner_members = NIL; - ListCell *lc1; + Relids top_parent_join_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_join_relids = find_relids_top_parents(root, join_relids); /* * First, scan the EC to identify member values that are computable at the @@ -1581,9 +1664,15 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * as well as to at least one input member, plus enforce at least one * outer-rel member equal to at least one inner-rel member. */ - foreach(lc1, ec->ec_members) + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + bms_is_subset(cur_em->em_relids, top_parent_join_relids)) + iterate_child_rel_equivalences(&it, root, ec, cur_em, join_relids); /* * We don't need to check explicitly for child EC members. This test @@ -1600,6 +1689,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, else new_members = lappend(new_members, cur_em); } + dispose_eclass_child_member_iterator(&it); /* * First, select the joinclause if needed. We can equate any one outer @@ -1617,6 +1707,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, Oid best_eq_op = InvalidOid; int best_score = -1; RestrictInfo *rinfo; + ListCell *lc1; foreach(lc1, outer_members) { @@ -1691,6 +1782,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *old_members = list_concat(outer_members, inner_members); EquivalenceMember *prev_em = NULL; RestrictInfo *rinfo; + ListCell *lc1; /* For now, arbitrarily take the first old_member as the one to use */ if (old_members) @@ -1698,7 +1790,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, foreach(lc1, new_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + cur_em = (EquivalenceMember *) lfirst(lc1); if (prev_em != NULL) { @@ -2396,6 +2488,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) if (matchleft && matchright) { cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); + Assert(!bms_is_empty(coal_em->em_relids)); return true; } @@ -2466,8 +2559,8 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); if (equal(item1, em->em_expr)) item1member = true; else if (equal(item2, em->em_expr)) @@ -2539,8 +2632,8 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); Var *var; - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* EM must be a Var, possibly with RelabelType */ var = (Var *) em->em_expr; @@ -2637,6 +2730,7 @@ add_child_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_rel->top_parent_relids; Relids child_relids = child_rel->relids; int i; + ListCell *lc; /* * EC merging should be complete already, so we can use the parent rel's @@ -2649,7 +2743,6 @@ add_child_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2662,15 +2755,9 @@ add_child_rel_equivalences(PlannerInfo *root, /* Sanity check eclass_indexes only contain ECs for parent_rel */ Assert(bms_is_subset(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2683,8 +2770,8 @@ add_child_rel_equivalences(PlannerInfo *root, * combinations of children. (But add_child_join_rel_equivalences * may add targeted combinations for partitionwise-join purposes.) */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * Consider only members that reference and can be computed at @@ -2700,6 +2787,7 @@ add_child_rel_equivalences(PlannerInfo *root, /* OK, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_rel->reloptkind == RELOPT_BASEREL) { @@ -2729,9 +2817,20 @@ add_child_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, + child_em); + + /* + * We save the knowledge that 'child_em' can be translated from + * 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + cur_em->em_child_relids = bms_add_member(cur_em->em_child_relids, + child_rel->relid); /* Record this EC index for the child rel */ child_rel->eclass_indexes = bms_add_member(child_rel->eclass_indexes, i); @@ -2760,6 +2859,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; MemoryContext oldcontext; + ListCell *lc; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2781,7 +2881,6 @@ add_child_join_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(matching_ecs, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2794,15 +2893,9 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Sanity check on get_eclass_indexes_for_relids result */ Assert(bms_overlap(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2811,8 +2904,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, * We consider only original EC members here, not * already-transformed child members. */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * We may ignore expressions that reference a single baserel, @@ -2827,6 +2920,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Yes, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_joinrel->reloptkind == RELOPT_JOINREL) { @@ -2857,9 +2951,21 @@ add_child_join_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_joinrel->eclass_child_members = + lappend(child_joinrel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated from + * 'child_joinrel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + cur_em->em_child_joinrel_relids = + bms_add_member(cur_em->em_child_joinrel_relids, + child_joinrel->join_rel_list_index); } } } @@ -2867,6 +2973,143 @@ add_child_join_rel_equivalences(PlannerInfo *root, MemoryContextSwitchTo(oldcontext); } +/* + * setup_eclass_child_member_iterator + * Setup an EquivalenceChildMemberIterator 'it' so that it can iterate over + * EquivalenceMembers in 'ec'. + * + * Once used, the caller should dispose of the iterator by calling + * dispose_eclass_child_member_iterator(). + */ +void +setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec) +{ + it->index = -1; + it->modified = false; + it->ec_members = ec->ec_members; +} + +/* + * eclass_child_member_iterator_next + * Get a next EquivalenceMember from an EquivalenceChildMemberIterator 'it' + * that was setup by setup_eclass_child_member_iterator(). NULL is returned + * if there are no members left. + */ +EquivalenceMember * +eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it) +{ + if (++it->index < list_length(it->ec_members)) + return list_nth_node(EquivalenceMember, it->ec_members, it->index); + return NULL; +} + +/* + * dispose_eclass_child_member_iterator + * Free any memory allocated by the iterator. + */ +void +dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it) +{ + /* + * XXX Should we use list_free()? I decided to use this style to take + * advantage of speculative execution. + */ + if (unlikely(it->modified)) + pfree(it->ec_members); +} + +/* + * add_transformed_child_version + * Add a transformed EquivalenceMember referencing the given child_rel to + * the iterator. + * + * This function is expected to be called only from + * iterate_child_rel_equivalences(). + */ +static void +add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel) +{ + ListCell *lc; + + foreach(lc, child_rel->eclass_child_members) + { + EquivalenceMember *child_em = lfirst_node(EquivalenceMember, lc); + + /* Skip unwanted EquivalenceMembers */ + if (child_em->em_parent != parent_em) + continue; + + /* + * If this is the first time the iterator's list has been modified, + * we need to make a copy of it. + */ + if (!it->modified) + { + it->ec_members = list_copy(it->ec_members); + it->modified = true; + } + + /* Add this child EquivalenceMember to the list */ + it->ec_members = lappend(it->ec_members, child_em); + } +} + +/* + * iterate_child_rel_equivalences + * Add transformed EquivalenceMembers referencing child rels in the given + * child_relids to the iterator. + * + * The transformation is done in add_transformed_child_version(). + */ +void +iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids) +{ + int i; + Relids matching_relids; + + /* The given EquivalenceMember should be parent */ + Assert(!parent_em->em_is_child); + + /* + * First, we translate simple rels. + */ + matching_relids = bms_intersect(parent_em->em_child_relids, + child_relids); + i = -1; + while ((i = bms_next_member(matching_relids, i)) >= 0) + { + /* Add transformed child version for this child rel */ + RelOptInfo *child_rel = root->simple_rel_array[i]; + + Assert(child_rel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_rel); + } + bms_free(matching_relids); + + /* + * Next, we try to translate join rels. + */ + i = -1; + while ((i = bms_next_member(parent_em->em_child_joinrel_relids, i)) >= 0) + { + /* Add transformed child version for this child join rel */ + RelOptInfo *child_joinrel = + list_nth_node(RelOptInfo, root->join_rel_list, i); + + Assert(child_joinrel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_joinrel); + } +} + /* * generate_implied_equalities_for_column @@ -2900,7 +3143,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, { List *result = NIL; bool is_child_rel = (rel->reloptkind == RELOPT_OTHER_MEMBER_REL); - Relids parent_relids; + Relids parent_relids, top_parent_rel_relids; int i; /* Should be OK to rely on eclass_indexes */ @@ -2909,6 +3152,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, /* Indexes are available only on base or "other" member relations. */ Assert(IS_SIMPLE_REL(rel)); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + /* If it's a child rel, we'll need to know what its parent(s) are */ if (is_child_rel) parent_relids = find_childrel_parents(root, rel); @@ -2921,6 +3167,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); EquivalenceMember *cur_em; ListCell *lc2; + EquivalenceChildMemberIterator it; /* Sanity check eclass_indexes only contain ECs for rel */ Assert(is_child_rel || bms_is_subset(rel->relids, cur_ec->ec_relids)); @@ -2942,15 +3189,19 @@ generate_implied_equalities_for_column(PlannerInfo *root, * corner cases, so for now we live with just reporting the first * match. See also get_eclass_for_sort_expr.) */ - cur_em = NULL; - foreach(lc2, cur_ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - cur_em = (EquivalenceMember *) lfirst(lc2); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel->relids); if (bms_equal(cur_em->em_relids, rel->relids) && callback(root, rel, cur_ec, cur_em, callback_arg)) break; - cur_em = NULL; } + dispose_eclass_child_member_iterator(&it); if (!cur_em) continue; @@ -2965,8 +3216,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, Oid eq_op; RestrictInfo *rinfo; - if (other_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!other_em->em_is_child); /* Make sure it'll be a join to a different rel */ if (other_em == cur_em || @@ -3184,8 +3435,8 @@ eclass_useful_for_merging(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); if (!bms_overlap(cur_em->em_relids, relids)) return true; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 32c6a8bbdc..b06c7cb389 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -184,7 +184,7 @@ static IndexClause *expand_indexqual_rowcompare(PlannerInfo *root, IndexOptInfo *index, Oid expr_op, bool var_on_left); -static void match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +static void match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p); static Expr *match_clause_to_ordering_op(IndexOptInfo *index, @@ -980,7 +980,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * query_pathkeys will allow an incremental sort to be considered on * the index's partially sorted results. */ - match_pathkeys_to_index(index, root->query_pathkeys, + match_pathkeys_to_index(root, index, root->query_pathkeys, &orderbyclauses, &orderbyclausecols); if (list_length(root->query_pathkeys) == list_length(orderbyclauses)) @@ -3071,12 +3071,16 @@ expand_indexqual_rowcompare(PlannerInfo *root, * item in the given 'pathkeys' list. */ static void -match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p) { + Relids top_parent_index_relids; ListCell *lc1; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_index_relids = find_relids_top_parents(root, index->rel->relids); + *orderby_clauses_p = NIL; /* set default results */ *clause_columns_p = NIL; @@ -3088,7 +3092,8 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, { PathKey *pathkey = (PathKey *) lfirst(lc1); bool found = false; - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *member; /* Pathkey must request default sort order for the target opfamily */ @@ -3108,15 +3113,30 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, * be considered to match more than one pathkey list, which is OK * here. See also get_eclass_for_sort_expr.) */ - foreach(lc2, pathkey->pk_eclass->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, pathkey->pk_eclass); + while ((member = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *member = (EquivalenceMember *) lfirst(lc2); int indexcol; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + bms_equal(member->em_relids, top_parent_index_relids)) + iterate_child_rel_equivalences(&it, root, pathkey->pk_eclass, member, + index->rel->relids); + /* No possibility of match if it references other relations */ - if (!bms_equal(member->em_relids, index->rel->relids)) + if (member->em_is_child || + !bms_equal(member->em_relids, index->rel->relids)) continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(bms_equal(member->em_relids, index->rel->relids)); + /* * We allow any column of the index to match each pathkey; they * don't have to match left-to-right as you might expect. This is @@ -3145,6 +3165,7 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, if (found) /* don't want to look at remaining members */ break; } + dispose_eclass_child_member_iterator(&it); /* * Return the matches found so far when this pathkey couldn't be diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index da6f457a3b..c32be3f8d2 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -1168,8 +1168,8 @@ convert_subquery_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid sub_expr_coll = sub_eclass->ec_collation; ListCell *k; - if (sub_member->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!sub_member->em_is_child); foreach(k, subquery_tlist) { @@ -1695,8 +1695,11 @@ select_outer_pathkeys_for_merge(PlannerInfo *root, { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); + /* Potential future join partner? */ - if (!em->em_is_const && !em->em_is_child && + if (!em->em_is_const && !bms_overlap(em->em_relids, joinrel->relids)) score++; } diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 4978758f8e..4e72741997 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -52,7 +52,7 @@ static void remove_leftjoinrel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid); -static void remove_rel_from_eclass(EquivalenceClass *ec, +static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static bool rel_supports_distinctness(PlannerInfo *root, RelOptInfo *rel); @@ -570,7 +570,7 @@ remove_leftjoinrel_from_query(PlannerInfo *root, int relid, if (bms_is_member(relid, ec->ec_relids) || bms_is_member(ojrelid, ec->ec_relids)) - remove_rel_from_eclass(ec, relid, ojrelid); + remove_rel_from_eclass(root, ec, relid, ojrelid); } /* @@ -655,7 +655,8 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) * level(s). */ static void -remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) +remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, + int ojrelid) { ListCell *lc; diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 610f4a56d6..59250f98f1 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -260,7 +260,9 @@ static IncrementalSort *make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); -static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -269,9 +271,11 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); -static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Sort *make_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids); -static IncrementalSort *make_incrementalsort_from_pathkeys(Plan *lefttree, +static IncrementalSort *make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, int nPresortedCols); static Sort *make_sort_from_groupcols(List *groupcls, AttrNumber *grpColIdx, @@ -293,7 +297,7 @@ static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); static Unique *make_unique_from_sortclauses(Plan *lefttree, List *distinctList); -static Unique *make_unique_from_pathkeys(Plan *lefttree, +static Unique *make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, int numCols); static Gather *make_gather(List *qptlist, List *qpqual, int nworkers, int rescan_param, bool single_copy, Plan *subplan); @@ -1280,7 +1284,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * function result; it must be the same plan node. However, we then * need to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys((Plan *) plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, (Plan *) plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1324,7 +1328,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * don't need an explicit sort, to make sure they are returning * the same sort key columns the Append expects. */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, nodeSortColIdx, false, @@ -1465,7 +1469,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * function result; it must be the same plan node. However, we then need * to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys(plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1496,7 +1500,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, subplan = create_plan_recurse(root, subpath, CP_EXACT_TLIST); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, node->sortColIdx, false, @@ -1975,7 +1979,7 @@ create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) Assert(pathkeys != NIL); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, best_path->subpath->parent->relids, gm_plan->sortColIdx, false, @@ -2194,7 +2198,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) * relids. Thus, if this sort path is based on a child relation, we must * pass its relids. */ - plan = make_sort_from_pathkeys(subplan, best_path->path.pathkeys, + plan = make_sort_from_pathkeys(root, subplan, best_path->path.pathkeys, IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -2218,7 +2222,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, /* See comments in create_sort_plan() above */ subplan = create_plan_recurse(root, best_path->spath.subpath, flags | CP_SMALL_TLIST); - plan = make_incrementalsort_from_pathkeys(subplan, + plan = make_incrementalsort_from_pathkeys(root, subplan, best_path->spath.path.pathkeys, IS_OTHER_REL(best_path->spath.subpath->parent) ? best_path->spath.path.parent->relids : NULL, @@ -2287,7 +2291,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag subplan = create_plan_recurse(root, best_path->subpath, flags | CP_LABEL_TLIST); - plan = make_unique_from_pathkeys(subplan, + plan = make_unique_from_pathkeys(root, subplan, best_path->path.pathkeys, best_path->numkeys); @@ -4507,7 +4511,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->outersortkeys) { Relids outer_relids = outer_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(outer_plan, + Sort *sort = make_sort_from_pathkeys(root, outer_plan, best_path->outersortkeys, outer_relids); @@ -4521,7 +4525,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->innersortkeys) { Relids inner_relids = inner_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(inner_plan, + Sort *sort = make_sort_from_pathkeys(root, inner_plan, best_path->innersortkeys, inner_relids); @@ -6142,7 +6146,7 @@ make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, * or a Result stacked atop lefttree). */ static Plan * -prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -6209,7 +6213,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ @@ -6237,7 +6241,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr already in tlist */ @@ -6253,7 +6257,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, /* * No matching tlist item; look for a computable expression. */ - em = find_computable_ec_member(NULL, ec, tlist, relids, false); + em = find_computable_ec_member(root, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; @@ -6324,7 +6328,8 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ static Sort * -make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) +make_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + Relids relids) { int numsortkeys; AttrNumber *sortColIdx; @@ -6333,7 +6338,7 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6359,8 +6364,9 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) * 'nPresortedCols' is the number of presorted columns in input tuples */ static IncrementalSort * -make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, - Relids relids, int nPresortedCols) +make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, + List *pathkeys, Relids relids, + int nPresortedCols) { int numsortkeys; AttrNumber *sortColIdx; @@ -6369,7 +6375,7 @@ make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6726,7 +6732,8 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) * as above, but use pathkeys to identify the sort columns and semantics */ static Unique * -make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) +make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + int numCols) { Unique *node = makeNode(Unique); Plan *plan = &node->plan; @@ -6789,7 +6796,7 @@ make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) foreach(j, plan->targetlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, NULL); + em = find_ec_member_matching_expr(root, ec, tle->expr, NULL); if (em) { /* found expr already in tlist */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 5c7acf8a90..3501fbbeed 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -461,6 +461,7 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, RelationGetRelid(parentrel); Oid childOID = RelationGetRelid(childrel); RangeTblEntry *childrte; + Index topParentRTindex; Index childRTindex; AppendRelInfo *appinfo; TupleDesc child_tupdesc; @@ -568,6 +569,19 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, Assert(root->append_rel_array[childRTindex] == NULL); root->append_rel_array[childRTindex] = appinfo; + /* + * Find a top parent rel's index and save it to top_parent_relid_array. + */ + if (root->top_parent_relid_array == NULL) + root->top_parent_relid_array = + (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); + Assert(root->top_parent_relid_array[childRTindex] == 0); + topParentRTindex = parentRTindex; + while (root->append_rel_array[topParentRTindex] != NULL && + root->append_rel_array[topParentRTindex]->parent_relid != 0) + topParentRTindex = root->append_rel_array[topParentRTindex]->parent_relid; + root->top_parent_relid_array[childRTindex] = topParentRTindex; + /* * Build a PlanRowMark if parent is marked FOR UPDATE/SHARE. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index e5f4062bfb..9973326e90 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -123,11 +123,14 @@ setup_simple_rel_arrays(PlannerInfo *root) if (root->append_rel_list == NIL) { root->append_rel_array = NULL; + root->top_parent_relid_array = NULL; return; } root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); + root->top_parent_relid_array = (Index *) + palloc0(size * sizeof(Index)); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -148,6 +151,27 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array[child_relid] = appinfo; } + + /* + * Find a top parent rel's relid for each AppendRelInfo. We cannot do this + * in the last foreach loop because there may be multi-level parent-child + * relations. + */ + for (int i = 0; i < size; i++) + { + int top_parent_relid; + + if (root->append_rel_array[i] == NULL) + continue; + + top_parent_relid = root->append_rel_array[i]->parent_relid; + + while (root->append_rel_array[top_parent_relid] != NULL && + root->append_rel_array[top_parent_relid]->parent_relid != 0) + top_parent_relid = root->append_rel_array[top_parent_relid]->parent_relid; + + root->top_parent_relid_array[i] = top_parent_relid; + } } /* @@ -175,11 +199,23 @@ expand_planner_arrays(PlannerInfo *root, int add_size) repalloc0_array(root->simple_rte_array, RangeTblEntry *, root->simple_rel_array_size, new_size); if (root->append_rel_array) + { root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); + root->top_parent_relid_array = + repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + } else + { root->append_rel_array = palloc0_array(AppendRelInfo *, new_size); + /* + * We do not allocate top_parent_relid_array here so that + * find_relids_top_parents() can early find all of the given Relids are + * top-level. + */ + root->top_parent_relid_array = NULL; + } root->simple_rel_array_size = new_size; } @@ -234,6 +270,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; + rel->eclass_child_members = NIL; rel->serverid = InvalidOid; if (rte->rtekind == RTE_RELATION) { @@ -622,6 +659,12 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) /* GEQO requires us to append the new joinrel to the end of the list! */ root->join_rel_list = lappend(root->join_rel_list, joinrel); + /* + * Store the index of this joinrel to use in + * add_child_join_rel_equivalences(). + */ + joinrel->join_rel_list_index = list_length(root->join_rel_list) - 1; + /* store it into the auxiliary hashtable if there is one. */ if (root->join_rel_hash) { @@ -734,6 +777,7 @@ build_join_rel(PlannerInfo *root, joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -931,6 +975,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -1495,6 +1540,7 @@ fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids) upperrel->cheapest_total_path = NULL; upperrel->cheapest_unique_path = NULL; upperrel->cheapest_parameterized_paths = NIL; + upperrel->eclass_child_members = NIL; /* XXX Is this required? */ root->upper_rels[kind] = lappend(root->upper_rels[kind], upperrel); @@ -1535,6 +1581,48 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } +/* + * find_relids_top_parents_slow + * Slow path of find_relids_top_parents() macro. + * + * Do not call this directly; use the macro instead. See the macro comment for + * more details. + */ +Relids +find_relids_top_parents_slow(PlannerInfo *root, Relids relids) +{ + Index *top_parent_relid_array = root->top_parent_relid_array; + Relids result; + bool is_top_parent; + int i; + + /* This function should be called in the slow path */ + Assert(top_parent_relid_array != NULL); + + result = NULL; + is_top_parent = true; + i = -1; + while ((i = bms_next_member(relids, i)) >= 0) + { + int top_parent_relid = (int) top_parent_relid_array[i]; + + if (top_parent_relid == 0) + top_parent_relid = i; /* 'i' has no parents, so add itself */ + else if (top_parent_relid != i) + is_top_parent = false; + result = bms_add_member(result, top_parent_relid); + } + + if (is_top_parent) + { + bms_free(result); + return NULL; + } + + return result; +} + + /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 534692bee1..c359505f6a 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -245,6 +245,14 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * append_rel_array is the same length as simple_rel_array and holds the + * top-level parent indexes of the corresponding rels within + * simple_rel_array. The element can be zero if the rel has no parents, + * i.e., is itself in a top-level. + */ + Index *top_parent_relid_array pg_node_attr(read_write_ignore); + /* * all_baserels is a Relids set of all base relids (but not joins or * "other" rels) in the query. This is computed in deconstruct_jointree. @@ -938,6 +946,17 @@ typedef struct RelOptInfo /* Bitmask of optional features supported by the table AM */ uint32 amflags; + /* + * information about a join rel + */ + /* index in root->join_rel_list of this rel */ + Index join_rel_list_index; + + /* + * EquivalenceMembers referencing this rel + */ + List *eclass_child_members; + /* * Information about foreign tables and foreign joins */ @@ -1424,10 +1443,46 @@ typedef struct EquivalenceMember bool em_is_child; /* derived version for a child relation? */ Oid em_datatype; /* the "nominal type" used by the opfamily */ JoinDomain *em_jdomain; /* join domain containing the source clause */ + Relids em_child_relids; /* all relids of child rels */ + Bitmapset *em_child_joinrel_relids; /* indexes in root->join_rel_list of + join rel children */ /* if em_is_child is true, this links to corresponding EM for top parent */ struct EquivalenceMember *em_parent pg_node_attr(read_write_ignore); } EquivalenceMember; +/* + * EquivalenceChildMemberIterator + * + * EquivalenceClass has only parent EquivalenceMembers, so we need to translate + * child members if necessary. EquivalenceChildMemberIterator provides a way to + * iterate over translated child members during the loop in addition to all of + * the parent EquivalenceMembers. + * + * The most common way to use this struct is as follows: + * ----- + * EquivalenceClass *ec = given; + * Relids rel = given; + * EquivalenceChildMemberIterator it; + * EquivalenceMember *em; + * + * setup_eclass_child_member_iterator(&it, ec); + * while ((em = eclass_child_member_iterator_next(&it)) != NULL) + * { + * use em ...; + * if (we need to iterate over child EquivalenceMembers) + * iterate_child_rel_equivalences(&it, root, ec, em, rel); + * use em ...; + * } + * dispose_eclass_child_member_iterator(&it); + * ----- + */ +typedef struct +{ + int index; /* current index within 'ec_members'. */ + bool modified; /* is 'ec_members' a newly allocated one? */ + List *ec_members; /* parent and child members*/ +} EquivalenceChildMemberIterator; + /* * PathKeys * diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index c43d97b48a..e61f2e3529 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -324,6 +324,24 @@ extern Relids min_join_parameterization(PlannerInfo *root, extern RelOptInfo *fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids); extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); + +/* + * find_relids_top_parents + * Compute the set of top-parent relids of rel. + * + * Replaces all Relids appearing in the given 'relids' as their top-level + * parents. The result will be NULL if and only if all of the given relids are + * top-level. + * + * The motivation for having this feature as a macro rather than a function is + * that Relids are top-level in most cases. We can quickly determine when + * root->top_parent_relid_array is NULL. + */ +#define find_relids_top_parents(root, relids) \ + (likely((root)->top_parent_relid_array == NULL) \ + ? NULL : find_relids_top_parents_slow(root, relids)) +extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); + extern ParamPathInfo *get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, Relids required_outer); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 0e8a9c94ba..5f6a08821e 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -137,7 +137,8 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Index sortref, Relids rel, bool create_it); -extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec, +extern EquivalenceMember *find_ec_member_matching_expr(PlannerInfo *root, + EquivalenceClass *ec, Expr *expr, Relids relids); extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root, @@ -174,6 +175,15 @@ extern void add_child_join_rel_equivalences(PlannerInfo *root, AppendRelInfo **appinfos, RelOptInfo *parent_joinrel, RelOptInfo *child_joinrel); +extern void setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec); +extern EquivalenceMember *eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it); +extern void dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it); +extern void iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids); extern List *generate_implied_equalities_for_column(PlannerInfo *root, RelOptInfo *rel, ec_matches_callback_type callback, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index fc8b15d0cf..8f86dd864e 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -661,6 +661,7 @@ EphemeralNamedRelation EphemeralNamedRelationData EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData +EquivalenceChildMemberIterator EquivalenceClass EquivalenceMember ErrorContextCallback -- 2.42.0.windows.2 [application/octet-stream] v24-0002-Introduce-indexes-for-RestrictInfo.patch (52.5K, ../../CAJ2pMkYRQxNRt=yjxebWxynNVw0onxDSbi4xv9iW3hnGcPcabg@mail.gmail.com/3-v24-0002-Introduce-indexes-for-RestrictInfo.patch) download | inline diff: From c2f94809f72a8c89b67326a000e5bbb6db52ffbf Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:44:25 +0900 Subject: [PATCH v24 2/3] Introduce indexes for RestrictInfo This commit adds indexes to speed up searches for RestrictInfos. When there are many child partitions and we have to perform planning for join operations on the tables, we have to handle a large number of RestrictInfos, resulting in a long planning time. This commit adds ec_[source|derive]_indexes to speed up the search. We can use the indexes to filter out unwanted RestrictInfos, and improve the planning performance. Author: David Rowley <[email protected]> and me, and includes rebase by Alena Rybakina <[email protected]> [1]. [1] https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/nodes/outfuncs.c | 6 +- src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/path/equivclass.c | 519 ++++++++++++++++++++-- src/backend/optimizer/plan/analyzejoins.c | 175 ++++++-- src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/appendinfo.c | 2 + src/backend/optimizer/util/inherit.c | 7 + src/backend/optimizer/util/restrictinfo.c | 5 + src/include/nodes/parsenodes.h | 6 + src/include/nodes/pathnodes.h | 41 +- src/include/optimizer/paths.h | 21 +- 13 files changed, 693 insertions(+), 98 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 2c30bba212..c61110c200 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -463,8 +463,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_opfamilies); WRITE_OID_FIELD(ec_collation); WRITE_NODE_FIELD(ec_members); - WRITE_NODE_FIELD(ec_sources); - WRITE_NODE_FIELD(ec_derives); + WRITE_BITMAPSET_FIELD(ec_source_indexes); + WRITE_BITMAPSET_FIELD(ec_derive_indexes); WRITE_BITMAPSET_FIELD(ec_relids); WRITE_BOOL_FIELD(ec_has_const); WRITE_BOOL_FIELD(ec_has_volatile); @@ -567,6 +567,8 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(inh); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); + WRITE_BITMAPSET_FIELD(eclass_source_indexes); + WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index b1e2f2b440..e3518c602b 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -431,6 +431,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); + READ_BITMAPSET_FIELD(eclass_source_indexes); + READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 8b76e98529..ae64b169f8 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5784,7 +5784,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, if (ec && ec->ec_has_const) { EquivalenceMember *em = fkinfo->fk_eclass_member[i]; - RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + RestrictInfo *rinfo = find_derived_clause_for_ec_member(root, + ec, em); if (rinfo) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 1931b01cbc..17fffc4087 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -42,6 +42,10 @@ static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, Oid datatype); +static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); +static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); @@ -320,7 +324,6 @@ process_equivalence(PlannerInfo *root, /* If case 1, nothing to do, except add to sources */ if (ec1 == ec2) { - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -331,6 +334,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); return true; } @@ -351,8 +356,10 @@ process_equivalence(PlannerInfo *root, * be found. */ ec1->ec_members = list_concat(ec1->ec_members, ec2->ec_members); - ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); - ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); + ec1->ec_source_indexes = bms_join(ec1->ec_source_indexes, + ec2->ec_source_indexes); + ec1->ec_derive_indexes = bms_join(ec1->ec_derive_indexes, + ec2->ec_derive_indexes); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); ec1->ec_has_const |= ec2->ec_has_const; /* can't need to set has_volatile */ @@ -364,10 +371,9 @@ process_equivalence(PlannerInfo *root, root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; - ec2->ec_sources = NIL; - ec2->ec_derives = NIL; + ec2->ec_source_indexes = NULL; + ec2->ec_derive_indexes = NULL; ec2->ec_relids = NULL; - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -378,13 +384,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec1) { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, jdomain, item2_type); - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -395,13 +402,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec2) { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, jdomain, item1_type); - ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -412,6 +420,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec2, restrictinfo); } else { @@ -421,8 +431,8 @@ process_equivalence(PlannerInfo *root, ec->ec_opfamilies = opfamilies; ec->ec_collation = collation; ec->ec_members = NIL; - ec->ec_sources = list_make1(restrictinfo); - ec->ec_derives = NIL; + ec->ec_source_indexes = NULL; + ec->ec_derive_indexes = NULL; ec->ec_relids = NULL; ec->ec_has_const = false; ec->ec_has_volatile = false; @@ -444,6 +454,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec, restrictinfo); } return true; @@ -578,6 +590,170 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, return em; } +/* + * add_eq_source - add 'rinfo' in eq_sources for this 'ec' + */ +static void +add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int source_idx = list_length(root->eq_sources); + int i; + + ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); + root->eq_sources = lappend(root->eq_sources, rinfo); + Assert(rinfo->eq_sources_index == -1); + rinfo->eq_sources_index = source_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + source_idx); + } +} + +/* + * add_eq_derive - add 'rinfo' in eq_derives for this 'ec' + */ +static void +add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int derive_idx = list_length(root->eq_derives); + int i; + + ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); + root->eq_derives = lappend(root->eq_derives, rinfo); + Assert(rinfo->eq_derives_index == -1); + rinfo->eq_derives_index = derive_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + derive_idx); + } +} + +/* + * update_clause_relids + * Update RestrictInfo->clause_relids with adjusting the relevant indexes. + * The clause_relids field must be updated through this function, not by + * setting the field directly. + */ +void +update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids) +{ + int i; + Relids removing_relids; + Relids adding_relids; +#ifdef USE_ASSERT_CHECKING + Relids common_relids; +#endif + + /* + * If there is nothing to do, we return immediately after setting the + * clause_relids field. + */ + if (rinfo->eq_sources_index == -1 && rinfo->eq_derives_index == -1) + { + rinfo->clause_relids = new_clause_relids; + return; + } + + /* + * Remove any references between this RestrictInfo and RangeTblEntries + * that are no longer relevant. + */ + removing_relids = bms_difference(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(removing_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(removing_relids); + + /* + * Add references between this RestrictInfo and RangeTblEntries that will + * be relevant. + */ + adding_relids = bms_difference(new_clause_relids, rinfo->clause_relids); + i = -1; + while ((i = bms_next_member(adding_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(!bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_add_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(!bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_add_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(adding_relids); + +#ifdef USE_ASSERT_CHECKING + /* + * Verify that all indexes are set for common Relids. + */ + common_relids = bms_intersect(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(common_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + } + } + bms_free(common_relids); +#endif + + /* + * Since we have done everything to adjust the indexes, we can set the + * clause_relids field. + */ + rinfo->clause_relids = new_clause_relids; +} + /* * get_eclass_for_sort_expr @@ -736,8 +912,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = collation; newec->ec_members = NIL; - newec->ec_sources = NIL; - newec->ec_derives = NIL; + newec->ec_source_indexes = NULL; + newec->ec_derive_indexes = NULL; newec->ec_relids = NULL; newec->ec_has_const = false; newec->ec_has_volatile = contain_volatile_functions((Node *) expr); @@ -1115,7 +1291,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources or ec_derives for matches. It doesn't really + * don't search eq_sources or eq_derives for matches. It doesn't really * seem worth the trouble to do so. */ void @@ -1204,6 +1380,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *const_em = NULL; ListCell *lc; + int i; /* * In the trivial case where we just had one "var = const" clause, push @@ -1213,9 +1390,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, * equivalent to the old one. */ if (list_length(ec->ec_members) == 2 && - list_length(ec->ec_sources) == 1) + bms_get_singleton_member(ec->ec_source_indexes, &i)) { - RestrictInfo *restrictinfo = (RestrictInfo *) linitial(ec->ec_sources); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, root->eq_sources, i); distribute_restrictinfo_to_rels(root, restrictinfo); return; @@ -1273,9 +1450,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the correct - * markings for a mergejoinable clause, and save it in ec_derives. (We + * markings for a mergejoinable clause, and save it in eq_derives. (We * will not re-use such clauses directly, but selectivity estimation - * may consult the list later. Note that this use of ec_derives does + * may consult the list later. Note that this use of eq_derives does * not overlap with its use for join clauses, since we never generate * join clauses from an ec_has_const eclass.) */ @@ -1285,7 +1462,8 @@ generate_base_implied_equalities_const(PlannerInfo *root, rinfo->left_ec = rinfo->right_ec = ec; rinfo->left_em = cur_em; rinfo->right_em = const_em; - ec->ec_derives = lappend(ec->ec_derives, rinfo); + + add_eq_derive(root, ec, rinfo); } } } @@ -1351,7 +1529,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the * correct markings for a mergejoinable clause. We don't put it - * in ec_derives however; we don't currently need to re-find such + * in eq_derives however; we don't currently need to re-find such * clauses, and we don't want to clutter that list with non-join * clauses. */ @@ -1407,11 +1585,12 @@ static void generate_base_implied_equalities_broken(PlannerInfo *root, EquivalenceClass *ec) { - ListCell *lc; + int i = -1; - foreach(lc, ec->ec_sources) + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); if (ec->ec_has_const || bms_membership(restrictinfo->required_relids) != BMS_MULTIPLE) @@ -1452,11 +1631,11 @@ generate_base_implied_equalities_broken(PlannerInfo *root, * Because the same join clauses are likely to be needed multiple times as * we consider different join paths, we avoid generating multiple copies: * whenever we select a particular pair of EquivalenceMembers to join, - * we check to see if the pair matches any original clause (in ec_sources) - * or previously-built clause (in ec_derives). This saves memory and allows - * re-use of information cached in RestrictInfos. We also avoid generating - * commutative duplicates, i.e. if the algorithm selects "a.x = b.y" but - * we already have "b.y = a.x", we return the existing clause. + * we check to see if the pair matches any original clause in root's + * eq_sources or previously-built clause (in root's eq_derives). This saves + * memory and allows re-use of information cached in RestrictInfos. We also + * avoid generating commutative duplicates, i.e. if the algorithm selects + * "a.x = b.y" but we already have "b.y = a.x", we return the existing clause. * * If we are considering an outer join, sjinfo is the associated OJ info, * otherwise it can be NULL. @@ -1835,12 +2014,16 @@ generate_join_implied_equalities_broken(PlannerInfo *root, Relids nominal_inner_relids, RelOptInfo *inner_rel) { + Bitmapset *matching_es; List *result = NIL; - ListCell *lc; + int i; - foreach(lc, ec->ec_sources) + matching_es = get_ec_source_indexes(root, ec, nominal_join_relids); + i = -1; + while ((i = bms_next_member(matching_es, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); Relids clause_relids = restrictinfo->required_relids; if (bms_is_subset(clause_relids, nominal_join_relids) && @@ -1852,12 +2035,12 @@ generate_join_implied_equalities_broken(PlannerInfo *root, /* * If we have to translate, just brute-force apply adjust_appendrel_attrs * to all the RestrictInfos at once. This will result in returning - * RestrictInfos that are not listed in ec_derives, but there shouldn't be + * RestrictInfos that are not listed in eq_derives, but there shouldn't be * any duplication, and it's a sufficiently narrow corner case that we * shouldn't sweat too much over it anyway. * * Since inner_rel might be an indirect descendant of the baserel - * mentioned in the ec_sources clauses, we have to be prepared to apply + * mentioned in the eq_sources clauses, we have to be prepared to apply * multiple levels of Var translation. */ if (IS_OTHER_REL(inner_rel) && result != NIL) @@ -1919,10 +2102,11 @@ create_join_clause(PlannerInfo *root, EquivalenceMember *rightem, EquivalenceClass *parent_ec) { + Bitmapset *matches; RestrictInfo *rinfo; RestrictInfo *parent_rinfo = NULL; - ListCell *lc; MemoryContext oldcontext; + int i; /* * Search to see if we already built a RestrictInfo for this pair of @@ -1933,9 +2117,12 @@ create_join_clause(PlannerInfo *root, * it's not identical, it'd better have the same effects, or the operator * families we're using are broken. */ - foreach(lc, ec->ec_sources) + matches = bms_int_members(get_ec_source_indexes_strict(root, ec, leftem->em_relids), + get_ec_source_indexes_strict(root, ec, rightem->em_relids)); + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -1946,9 +2133,13 @@ create_join_clause(PlannerInfo *root, return rinfo; } - foreach(lc, ec->ec_derives) + matches = bms_int_members(get_ec_derive_indexes_strict(root, ec, leftem->em_relids), + get_ec_derive_indexes_strict(root, ec, rightem->em_relids)); + + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -2006,7 +2197,7 @@ create_join_clause(PlannerInfo *root, rinfo->left_em = leftem; rinfo->right_em = rightem; /* and save it for possible re-use */ - ec->ec_derives = lappend(ec->ec_derives, rinfo); + add_eq_derive(root, ec, rinfo); MemoryContextSwitchTo(oldcontext); @@ -2682,16 +2873,19 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, * Returns NULL if no such clause can be found. */ RestrictInfo * -find_derived_clause_for_ec_member(EquivalenceClass *ec, +find_derived_clause_for_ec_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em) { - ListCell *lc; + int i; Assert(ec->ec_has_const); Assert(!em->em_is_const); - foreach(lc, ec->ec_derives) + + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); /* * generate_base_implied_equalities_const will have put non-const @@ -3325,7 +3519,7 @@ have_relevant_eclass_joinclause(PlannerInfo *root, * surely be true if both of them overlap ec_relids.) * * Note we don't test ec_broken; if we did, we'd need a separate code - * path to look through ec_sources. Checking the membership anyway is + * path to look through eq_sources. Checking the membership anyway is * OK as a possibly-overoptimistic heuristic. * * We don't test ec_has_const either, even though a const eclass won't @@ -3413,7 +3607,7 @@ eclass_useful_for_merging(PlannerInfo *root, /* * Note we don't test ec_broken; if we did, we'd need a separate code path - * to look through ec_sources. Checking the members anyway is OK as a + * to look through eq_sources. Checking the members anyway is OK as a * possibly-overoptimistic heuristic. */ @@ -3566,3 +3760,240 @@ get_common_eclass_indexes(PlannerInfo *root, Relids relids1, Relids relids2) /* Calculate and return the common EC indexes, recycling the left input. */ return bms_int_members(rel1ecs, rel2ecs); } + +/* + * get_ec_source_indexes + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_esis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_esis, ec->ec_source_indexes); +} + +/* + * get_ec_source_indexes_strict + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *esis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + esis = bms_intersect(ec->ec_source_indexes, + rte->eclass_source_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + esis = bms_int_members(esis, rte->eclass_source_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return esis; +} + +/* + * get_ec_derive_indexes + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + * + * XXX is this function even needed? + */ +Bitmapset * +get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_edis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_edis, ec->ec_derive_indexes); +} + +/* + * get_ec_derive_indexes_strict + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *edis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + edis = bms_intersect(ec->ec_derive_indexes, + rte->eclass_derive_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + edis = bms_int_members(edis, rte->eclass_derive_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return edis; +} + +#ifdef USE_ASSERT_CHECKING +/* + * verify_eclass_indexes + * Verify that there are no missing references between RestrictInfos and + * EquivalenceMember's indexes, namely eclass_source_indexes and + * eclass_derive_indexes. If you modify these indexes, you should check + * them with this function. + */ +void +verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) +{ + ListCell *lc; + + /* + * All RestrictInfos in root->eq_sources must have references to + * eclass_source_indexes. + */ + foreach(lc, root->eq_sources) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_sources, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + } + } + + /* + * All RestrictInfos in root->eq_derives must have references to + * eclass_derive_indexes. + */ + foreach(lc, root->eq_derives) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_derives, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + } + } +} +#endif diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 4e72741997..13e987493b 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -50,7 +50,8 @@ static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); static void remove_leftjoinrel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); -static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, +static void remove_rel_from_restrictinfo(PlannerInfo *root, + RestrictInfo *rinfo, int relid, int ojrelid); static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); @@ -66,7 +67,7 @@ static bool is_innerrel_unique_for(PlannerInfo *root, JoinType jointype, List *restrictlist, List **extra_clauses); -static void replace_varno(Node *node, int from, int to); +static void replace_varno(PlannerInfo *root, Node *node, int from, int to); static Bitmapset *replace_relid(Relids relids, int oldId, int newId); static int self_join_candidates_cmp(const void *a, const void *b); @@ -385,7 +386,7 @@ remove_rel_from_query(PlannerInfo *root, RelOptInfo *rel, } /* Update lateral_vars list. */ - replace_varno((Node *) otherrel->lateral_vars, relid, subst); + replace_varno(root, (Node *) otherrel->lateral_vars, relid, subst); } /* @@ -422,7 +423,7 @@ remove_rel_from_query(PlannerInfo *root, RelOptInfo *rel, sjinf->commute_below_l = replace_relid(sjinf->commute_below_l, ojrelid, subst); sjinf->commute_below_r = replace_relid(sjinf->commute_below_r, ojrelid, subst); - replace_varno((Node *) sjinf->semi_rhs_exprs, relid, subst); + replace_varno(root, (Node *) sjinf->semi_rhs_exprs, relid, subst); } /* @@ -467,7 +468,7 @@ remove_rel_from_query(PlannerInfo *root, RelOptInfo *rel, phv->phrels = replace_relid(phv->phrels, relid, subst); phv->phrels = replace_relid(phv->phrels, ojrelid, subst); Assert(!bms_is_empty(phv->phrels)); - replace_varno((Node *) phv->phexpr, relid, subst); + replace_varno(root, (Node *) phv->phexpr, relid, subst); Assert(phv->phnullingrels == NULL); /* no need to adjust */ } } @@ -541,7 +542,7 @@ remove_leftjoinrel_from_query(PlannerInfo *root, int relid, * that any such PHV is safe (and updated its ph_eval_at), so we * can just drop those references. */ - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); /* * Cross-check that the clause itself does not reference the @@ -598,16 +599,24 @@ remove_leftjoinrel_from_query(PlannerInfo *root, int relid, * we have to also clean up the sub-clauses. */ static void -remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) +remove_rel_from_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo, int relid, + int ojrelid) { + Relids new_clause_relids; + + /* + * The 'new_clause_relids' passed to update_clause_relids() must be a + * different instance from the current rinfo->clause_relids, so we make a + * copy of them. + */ + new_clause_relids = bms_copy(rinfo->clause_relids); + new_clause_relids = bms_del_member(new_clause_relids, relid); + new_clause_relids = bms_del_member(new_clause_relids, ojrelid); + update_clause_relids(root, rinfo, new_clause_relids); /* - * The clause_relids probably aren't shared with anything else, but let's + * The required_relids probably aren't shared with anything else, but let's * copy them just to be sure. */ - rinfo->clause_relids = bms_copy(rinfo->clause_relids); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, relid); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, ojrelid); - /* Likewise for required_relids */ rinfo->required_relids = bms_copy(rinfo->required_relids); rinfo->required_relids = bms_del_member(rinfo->required_relids, relid); rinfo->required_relids = bms_del_member(rinfo->required_relids, ojrelid); @@ -632,14 +641,14 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) { RestrictInfo *rinfo2 = lfirst_node(RestrictInfo, lc2); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } else { RestrictInfo *rinfo2 = castNode(RestrictInfo, orarg); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } } @@ -659,6 +668,7 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid) { ListCell *lc; + int i; /* Fix up the EC's overall relids */ ec->ec_relids = bms_del_member(ec->ec_relids, relid); @@ -685,11 +695,12 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, } /* Fix up the source clauses, in case we can re-use them later */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); } /* @@ -697,7 +708,7 @@ remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, * drop them. (At this point, any such clauses would be base restriction * clauses, which we'd not need anymore anyway.) */ - ec->ec_derives = NIL; + ec->ec_derive_indexes = NULL; } /* @@ -1436,6 +1447,7 @@ is_innerrel_unique_for(PlannerInfo *root, typedef struct { + PlannerInfo *root; int from; int to; int sublevels_up; @@ -1494,10 +1506,10 @@ replace_varno_walker(Node *node, ReplaceVarnoContext *ctx) if (bms_is_member(ctx->from, rinfo->clause_relids)) { - replace_varno((Node *) rinfo->clause, ctx->from, ctx->to); - replace_varno((Node *) rinfo->orclause, ctx->from, ctx->to); - rinfo->clause_relids = - replace_relid(rinfo->clause_relids, ctx->from, ctx->to); + replace_varno(ctx->root, (Node *) rinfo->clause, ctx->from, ctx->to); + replace_varno(ctx->root, (Node *) rinfo->orclause, ctx->from, ctx->to); + update_clause_relids(ctx->root, rinfo, + replace_relid(rinfo->clause_relids, ctx->from, ctx->to)); rinfo->left_relids = replace_relid(rinfo->left_relids, ctx->from, ctx->to); rinfo->right_relids = @@ -1547,13 +1559,14 @@ replace_varno_walker(Node *node, ReplaceVarnoContext *ctx) } static void -replace_varno(Node *node, int from, int to) +replace_varno(PlannerInfo *root, Node *node, int from, int to) { ReplaceVarnoContext ctx; if (to <= 0) return; + ctx.root = root; ctx.from = from; ctx.to = to; ctx.sublevels_up = 0; @@ -1614,10 +1627,12 @@ replace_relid(Relids relids, int oldId, int newId) * delete them. */ static void -update_eclasses(EquivalenceClass *ec, int from, int to) +update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) { List *new_members = NIL; - List *new_sources = NIL; + Bitmapset *new_source_indexes = NULL; + int i; + int j; ListCell *lc; ListCell *lc1; @@ -1636,7 +1651,7 @@ update_eclasses(EquivalenceClass *ec, int from, int to) em->em_jdomain->jd_relids = replace_relid(em->em_jdomain->jd_relids, from, to); /* We only process inner joins */ - replace_varno((Node *) em->em_expr, from, to); + replace_varno(root, (Node *) em->em_expr, from, to); foreach(lc1, new_members) { @@ -1659,31 +1674,64 @@ update_eclasses(EquivalenceClass *ec, int from, int to) list_free(ec->ec_members); ec->ec_members = new_members; - list_free(ec->ec_derives); - ec->ec_derives = NULL; + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); + + /* + * Remove all references between this RestrictInfo and its relating + * RangeTblEntry. + */ + j = -1; + while ((j = bms_next_member(rinfo->clause_relids, j)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[j]; + + Assert(bms_is_member(i, rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, i); + } + + /* + * Can't delete the element because we would need to rebuild all + * the eq_derives indexes. But set a null to detect potential problems. + */ + list_nth_cell(root->eq_derives, i)->ptr_value = NULL; + + /* + * Since this RestrictInfo no longer exists in root->eq_derives, we + * must reset the stored index. + */ + rinfo->eq_derives_index = -1; + } + bms_free(ec->ec_derive_indexes); + ec->ec_derive_indexes = NULL; /* Update EC source expressions */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); bool is_redundant = false; if (!bms_is_member(from, rinfo->required_relids)) { - new_sources = lappend(new_sources, rinfo); + new_source_indexes = bms_add_member(new_source_indexes, i); continue; } - replace_varno((Node *) rinfo, from, to); + replace_varno(root, (Node *) rinfo, from, to); /* * After switching the clause to the remaining relation, check it for * redundancy with existing ones. We don't have to check for * redundancy with derived clauses, because we've just deleted them. */ - foreach(lc1, new_sources) + j = -1; + while ((j = bms_next_member(new_source_indexes, j)) >= 0) { - RestrictInfo *other = lfirst_node(RestrictInfo, lc1); + RestrictInfo *other = list_nth_node(RestrictInfo, root->eq_sources, j); if (!equal(rinfo->clause_relids, other->clause_relids)) continue; @@ -1695,13 +1743,46 @@ update_eclasses(EquivalenceClass *ec, int from, int to) } } - if (!is_redundant) - new_sources = lappend(new_sources, rinfo); + if (is_redundant) + { + /* + * Remove all references between this RestrictInfo and its relating + * RangeTblEntry. + */ + j = -1; + while ((j = bms_next_member(rinfo->clause_relids, j)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[j]; + + Assert(bms_is_member(i, rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, i); + } + + /* + * Can't delete the element because we would need to rebuild all + * the eq_sources indexes. But set a null to detect potential problems. + */ + list_nth_cell(root->eq_sources, i)->ptr_value = NULL; + + /* + * Since this RestrictInfo no longer exists in root->eq_sources, we + * must reset the stored index. + */ + rinfo->eq_sources_index = -1; + } + else + new_source_indexes = bms_add_member(new_source_indexes, i); } - list_free(ec->ec_sources); - ec->ec_sources = new_sources; + bms_free(ec->ec_source_indexes); + ec->ec_source_indexes = new_source_indexes; ec->ec_relids = replace_relid(ec->ec_relids, from, to); + +#ifdef USE_ASSERT_CHECKING + /* Make sure that we didn't break EquivalenceClass indexes */ + verify_eclass_indexes(root, ec); +#endif } /* @@ -1771,7 +1852,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); remove_join_clause_from_rels(root, rinfo, rinfo->required_relids); - replace_varno((Node *) rinfo, toRemove->relid, toKeep->relid); + replace_varno(root, (Node *) rinfo, toRemove->relid, toKeep->relid); if (bms_membership(rinfo->required_relids) == BMS_MULTIPLE) jinfo_candidates = lappend(jinfo_candidates, rinfo); @@ -1791,7 +1872,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); - replace_varno((Node *) rinfo, toRemove->relid, toKeep->relid); + replace_varno(root, (Node *) rinfo, toRemove->relid, toKeep->relid); if (bms_membership(rinfo->required_relids) == BMS_MULTIPLE) jinfo_candidates = lappend(jinfo_candidates, rinfo); @@ -1879,7 +1960,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, { EquivalenceClass *ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - update_eclasses(ec, toRemove->relid, toKeep->relid); + update_eclasses(root, ec, toRemove->relid, toKeep->relid); toKeep->eclass_indexes = bms_add_member(toKeep->eclass_indexes, i); } @@ -1891,7 +1972,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, { Node *node = lfirst(lc); - replace_varno(node, toRemove->relid, toKeep->relid); + replace_varno(root, node, toRemove->relid, toKeep->relid); if (!list_member(toKeep->reltarget->exprs, node)) toKeep->reltarget->exprs = lappend(toKeep->reltarget->exprs, node); } @@ -1932,7 +2013,7 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, } /* Replace varno in all the query structures */ - replace_varno((Node *) root->parse, toRemove->relid, toKeep->relid); + replace_varno(root, (Node *) root->parse, toRemove->relid, toKeep->relid); /* See remove_self_joins_one_group() */ Assert(root->parse->resultRelation != toRemove->relid); @@ -1942,9 +2023,9 @@ remove_self_join_rel(PlannerInfo *root, PlanRowMark *kmark, PlanRowMark *rmark, remove_rel_from_query(root, toRemove, toKeep->relid, NULL, NULL); /* At last, replace varno in root targetlist and HAVING clause */ - replace_varno((Node *) root->processed_tlist, + replace_varno(root, (Node *) root->processed_tlist, toRemove->relid, toKeep->relid); - replace_varno((Node *) root->processed_groupClause, + replace_varno(root, (Node *) root->processed_groupClause, toRemove->relid, toKeep->relid); replace_relid(root->all_result_relids, toRemove->relid, toKeep->relid); replace_relid(root->leaf_result_relids, toRemove->relid, toKeep->relid); @@ -2021,7 +2102,7 @@ split_selfjoin_quals(PlannerInfo *root, List *joinquals, List **selfjoinquals, * when we have cast of the same var to different (but compatible) * types. */ - replace_varno(rightexpr, bms_singleton_member(rinfo->right_relids), + replace_varno(root, rightexpr, bms_singleton_member(rinfo->right_relids), bms_singleton_member(rinfo->left_relids)); if (equal(leftexpr, rightexpr)) @@ -2062,7 +2143,7 @@ match_unique_clauses(PlannerInfo *root, RelOptInfo *outer, List *uclauses, bms_is_empty(rinfo->right_relids)); clause = (Expr *) copyObject(rinfo->clause); - replace_varno((Node *) clause, relid, outer->relid); + replace_varno(root, (Node *) clause, relid, outer->relid); iclause = bms_is_empty(rinfo->left_relids) ? get_rightop(clause) : get_leftop(clause); diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index be4e182869..1436cdd6e3 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -648,6 +648,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, root->multiexpr_params = NIL; root->join_domains = NIL; root->eq_classes = NIL; + root->eq_sources = NIL; + root->eq_derives = NIL; root->ec_merging_done = false; root->last_rinfo_serial = 0; root->all_result_relids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index aa83dd3636..9fc0b69580 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -993,6 +993,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, subroot->multiexpr_params = NIL; subroot->join_domains = NIL; subroot->eq_classes = NIL; + subroot->eq_sources = NIL; + subroot->eq_derives = NIL; subroot->ec_merging_done = false; subroot->last_rinfo_serial = 0; subroot->all_result_relids = NULL; diff --git a/src/backend/optimizer/util/appendinfo.c b/src/backend/optimizer/util/appendinfo.c index 51fdeace7d..27e4b093d7 100644 --- a/src/backend/optimizer/util/appendinfo.c +++ b/src/backend/optimizer/util/appendinfo.c @@ -491,6 +491,8 @@ adjust_appendrel_attrs_mutator(Node *node, newinfo->right_bucketsize = -1; newinfo->left_mcvfreq = -1; newinfo->right_mcvfreq = -1; + newinfo->eq_sources_index = -1; + newinfo->eq_derives_index = -1; return (Node *) newinfo; } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3501fbbeed..a65a76ace9 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -483,6 +483,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); + /* + * We do not want to inherit the EquivalenceMember indexes of the parent + * to its child + */ + childrte->eclass_source_indexes = NULL; + childrte->eclass_derive_indexes = NULL; + Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 0b406e9334..59ba503ecb 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -247,6 +247,9 @@ make_restrictinfo_internal(PlannerInfo *root, restrictinfo->left_hasheqoperator = InvalidOid; restrictinfo->right_hasheqoperator = InvalidOid; + restrictinfo->eq_sources_index = -1; + restrictinfo->eq_derives_index = -1; + return restrictinfo; } @@ -403,6 +406,8 @@ commute_restrictinfo(RestrictInfo *rinfo, Oid comm_op) result->right_mcvfreq = rinfo->left_mcvfreq; result->left_hasheqoperator = InvalidOid; result->right_hasheqoperator = InvalidOid; + result->eq_sources_index = -1; + result->eq_derives_index = -1; return result; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index baa6a97c7e..f624f41d99 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1194,6 +1194,12 @@ typedef struct RangeTblEntry bool inh; /* inheritance requested? */ bool inFromCl; /* present in FROM clause? */ List *securityQuals; /* security barrier quals to apply, if any */ + Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources + * list for RestrictInfos that mention + * this relation */ + Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives + * list for RestrictInfos that mention + * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index c359505f6a..81253d0351 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -318,6 +318,12 @@ struct PlannerInfo /* list of active EquivalenceClasses */ List *eq_classes; + /* list of source RestrictInfos used to build EquivalenceClasses */ + List *eq_sources; + + /* list of RestrictInfos derived from EquivalenceClasses */ + List *eq_derives; + /* set true once ECs are canonical */ bool ec_merging_done; @@ -1371,6 +1377,24 @@ typedef struct JoinDomain * entry: consider SELECT random() AS a, random() AS b ... ORDER BY b,a. * So we record the SortGroupRef of the originating sort clause. * + * At various locations in the query planner, we must search for source and + * derived RestrictInfos regarding a given EquivalenceClass. For the common + * case, an EquivalenceClass does not have a large number of RestrictInfos, + * however, in cases such as planning queries to partitioned tables, the + * number of members can become large. To maintain planning performance, we + * make use of a bitmap index to allow us to quickly find RestrictInfos in a + * given EquivalenceClass belonging to a given relation or set of relations. + * This is done by storing a list of RestrictInfos belonging to all + * EquivalenceClasses in PlannerInfo and storing a Bitmapset for each + * RelOptInfo which has a bit set for each RestrictInfo in that list which + * relates to the given relation. We also store a Bitmapset to mark all of + * the indexes in the PlannerInfo's list of RestrictInfos in the + * EquivalenceClass. We can quickly find the interesting indexes into the + * PlannerInfo's list by performing a bit-wise AND on the RelOptInfo's + * Bitmapset and the EquivalenceClasses. RestrictInfos must be looked up in + * PlannerInfo by this technique using the ec_source_indexes and + * ec_derive_indexes Bitmapsets. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1389,8 +1413,10 @@ typedef struct EquivalenceClass List *ec_opfamilies; /* btree operator family OIDs */ Oid ec_collation; /* collation, if datatypes are collatable */ List *ec_members; /* list of EquivalenceMembers */ - List *ec_sources; /* list of generating RestrictInfos */ - List *ec_derives; /* list of derived RestrictInfos */ + Bitmapset *ec_source_indexes; /* indexes into PlannerInfo's eq_sources + * list of generating RestrictInfos */ + Bitmapset *ec_derive_indexes; /* indexes into PlannerInfo's eq_derives + * list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except * for child members (see below) */ bool ec_has_const; /* any pseudoconstants in ec_members? */ @@ -2620,7 +2646,12 @@ typedef struct RestrictInfo /* number of base rels in clause_relids */ int num_base_rels pg_node_attr(equal_ignore); - /* The relids (varnos+varnullingrels) actually referenced in the clause: */ + /* + * The relids (varnos+varnullingrels) actually referenced in the clause. + * + * NOTE: This field must be updated through update_clause_relids(), not by + * setting the field directly. + */ Relids clause_relids pg_node_attr(equal_ignore); /* The set of relids required to evaluate the clause: */ @@ -2738,6 +2769,10 @@ typedef struct RestrictInfo /* hash equality operators used for memoize nodes, else InvalidOid */ Oid left_hasheqoperator pg_node_attr(equal_ignore); Oid right_hasheqoperator pg_node_attr(equal_ignore); + + /* the index within root->eq_sources and root->eq_derives */ + int eq_sources_index pg_node_attr(equal_ignore); + int eq_derives_index pg_node_attr(equal_ignore); } RestrictInfo; /* diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 5f6a08821e..6bb740266b 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -129,6 +129,8 @@ extern bool process_equivalence(PlannerInfo *root, extern Expr *canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation); extern void reconsider_outer_join_clauses(PlannerInfo *root); +extern void update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids); extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Expr *expr, List *opfamilies, @@ -164,7 +166,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2); extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); -extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, +extern RestrictInfo *find_derived_clause_for_ec_member(PlannerInfo *root, + EquivalenceClass *ec, EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, @@ -199,6 +202,22 @@ extern bool eclass_useful_for_merging(PlannerInfo *root, extern bool is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist); extern bool is_redundant_with_indexclauses(RestrictInfo *rinfo, List *indexclauses); +extern Bitmapset *get_ec_source_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_source_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +#ifdef USE_ASSERT_CHECKING +extern void verify_eclass_indexes(PlannerInfo *root, + EquivalenceClass *ec); +#endif /* * pathkeys.c -- 2.42.0.windows.2 [application/octet-stream] v24-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch (15.8K, ../../CAJ2pMkYRQxNRt=yjxebWxynNVw0onxDSbi4xv9iW3hnGcPcabg@mail.gmail.com/4-v24-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch) download | inline diff: From 79d246c59b1b6bc077c44805c7e1d7d1c190828a Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 19 Jan 2024 11:43:29 +0900 Subject: [PATCH v24 3/3] Move EquivalenceClass indexes to PlannerInfo In the previous commit, the indexes, namely ec_[source|derive]_indexes, were in RestrictInfo. This was a workaround because RelOptInfo was the best place but some RelOptInfos can be NULL and we cannot store indexes for them. This commit introduces a new struct, EquivalenceClassIndexes. This struct exists in PlannerInfo and holds our indexes. This change prevents a serialization problem with RestirctInfo in the previous commit. --- src/backend/nodes/outfuncs.c | 2 - src/backend/nodes/readfuncs.c | 2 - src/backend/optimizer/path/equivclass.c | 83 +++++++++++------------ src/backend/optimizer/plan/analyzejoins.c | 16 ++--- src/backend/optimizer/util/inherit.c | 7 -- src/backend/optimizer/util/relnode.c | 6 ++ src/include/nodes/parsenodes.h | 6 -- src/include/nodes/pathnodes.h | 24 +++++++ src/tools/pgindent/typedefs.list | 1 + 9 files changed, 80 insertions(+), 67 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index c61110c200..b08a4789c4 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -567,8 +567,6 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(inh); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); - WRITE_BITMAPSET_FIELD(eclass_source_indexes); - WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index e3518c602b..b1e2f2b440 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -431,8 +431,6 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(inh); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_BITMAPSET_FIELD(eclass_source_indexes); - READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 17fffc4087..4fb1ecff03 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -607,10 +607,10 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, - source_idx); + index->source_indexes = bms_add_member(index->source_indexes, + source_idx); } } @@ -631,10 +631,10 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, - derive_idx); + index->derive_indexes = bms_add_member(index->derive_indexes, + derive_idx); } } @@ -673,22 +673,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(removing_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_del_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_del_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_del_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_del_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -702,22 +702,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(adding_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(!bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_add_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_add_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(!bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_add_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_add_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -731,17 +731,17 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(common_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); + index->source_indexes)); } if (rinfo->eq_derives_index != -1) { Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); + index->derive_indexes)); } } bms_free(common_relids); @@ -3777,9 +3777,9 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + rel_esis = bms_add_members(rel_esis, index->source_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3815,7 +3815,7 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3824,12 +3824,12 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ esis = bms_intersect(ec->ec_source_indexes, - rte->eclass_source_indexes); + index->source_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - esis = bms_int_members(esis, rte->eclass_source_indexes); + index = &root->eclass_indexes_array[i]; + esis = bms_int_members(esis, index->source_indexes); } } @@ -3866,9 +3866,9 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + rel_edis = bms_add_members(rel_edis, index->derive_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3904,7 +3904,7 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3913,12 +3913,12 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, - rte->eclass_derive_indexes); + index->derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - edis = bms_int_members(edis, rte->eclass_derive_indexes); + index = &root->eclass_indexes_array[i]; + edis = bms_int_members(edis, index->derive_indexes); } } @@ -3941,9 +3941,8 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, /* * verify_eclass_indexes * Verify that there are no missing references between RestrictInfos and - * EquivalenceMember's indexes, namely eclass_source_indexes and - * eclass_derive_indexes. If you modify these indexes, you should check - * them with this function. + * EquivalenceMember's indexes, namely source_indexes and derive_indexes. + * If you modify these indexes, you should check them with this function. */ void verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) @@ -3952,7 +3951,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) /* * All RestrictInfos in root->eq_sources must have references to - * eclass_source_indexes. + * source_indexes. */ foreach(lc, root->eq_sources) { @@ -3969,13 +3968,13 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].source_indexes)); } } /* * All RestrictInfos in root->eq_derives must have references to - * eclass_derive_indexes. + * derive_indexes. */ foreach(lc, root->eq_derives) { @@ -3992,7 +3991,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].derive_indexes)); } } } diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 13e987493b..d2fdc2649a 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -1686,11 +1686,11 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) j = -1; while ((j = bms_next_member(rinfo->clause_relids, j)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[j]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[j]; - Assert(bms_is_member(i, rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_del_member(rte->eclass_derive_indexes, i); + Assert(bms_is_member(i, index->derive_indexes)); + index->derive_indexes = + bms_del_member(index->derive_indexes, i); } /* @@ -1752,11 +1752,11 @@ update_eclasses(PlannerInfo *root, EquivalenceClass *ec, int from, int to) j = -1; while ((j = bms_next_member(rinfo->clause_relids, j)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[j]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[j]; - Assert(bms_is_member(i, rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_del_member(rte->eclass_source_indexes, i); + Assert(bms_is_member(i, index->source_indexes)); + index->source_indexes = + bms_del_member(index->source_indexes, i); } /* diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index a65a76ace9..3501fbbeed 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -483,13 +483,6 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); - /* - * We do not want to inherit the EquivalenceMember indexes of the parent - * to its child - */ - childrte->eclass_source_indexes = NULL; - childrte->eclass_derive_indexes = NULL; - Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 9973326e90..bc9195c623 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -119,6 +119,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->simple_rte_array[rti++] = rte; } + root->eclass_indexes_array = (EquivalenceClassIndexes *) + palloc0(size * sizeof(EquivalenceClassIndexes)); + /* append_rel_array is not needed if there are no AppendRelInfos */ if (root->append_rel_list == NIL) { @@ -217,6 +220,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->top_parent_relid_array = NULL; } + root->eclass_indexes_array = + repalloc0_array(root->eclass_indexes_array, EquivalenceClassIndexes, root->simple_rel_array_size, new_size); + root->simple_rel_array_size = new_size; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index f624f41d99..baa6a97c7e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1194,12 +1194,6 @@ typedef struct RangeTblEntry bool inh; /* inheritance requested? */ bool inFromCl; /* present in FROM clause? */ List *securityQuals; /* security barrier quals to apply, if any */ - Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources - * list for RestrictInfos that mention - * this relation */ - Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives - * list for RestrictInfos that mention - * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 81253d0351..49f53fd51e 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -189,6 +189,8 @@ typedef struct PlannerInfo PlannerInfo; #define HAVE_PLANNERINFO_TYPEDEF 1 #endif +struct EquivalenceClassIndexes; + struct PlannerInfo { pg_node_attr(no_copy_equal, no_read, no_query_jumble) @@ -245,6 +247,13 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * eclass_indexes_array is the same length as simple_rel_array and holds + * the indexes of the corresponding rels for faster lookups of + * RestrictInfo. See the EquivalenceClass comment for more details. + */ + struct EquivalenceClassIndexes *eclass_indexes_array pg_node_attr(read_write_ignore); + /* * append_rel_array is the same length as simple_rel_array and holds the * top-level parent indexes of the corresponding rels within @@ -1509,6 +1518,21 @@ typedef struct List *ec_members; /* parent and child members*/ } EquivalenceChildMemberIterator; +/* + * EquivalenceClassIndexes + * + * As mentioned in the EquivalenceClass comment, we introduce a + * bitmapset-based indexing mechanism for faster lookups of RestrictInfo. This + * struct exists for each relation and holds the corresponding indexes. + */ +typedef struct EquivalenceClassIndexes +{ + Bitmapset *source_indexes; /* Indexes in PlannerInfo's eq_sources list for + RestrictInfos that mention this relation */ + Bitmapset *derive_indexes; /* Indexes in PlannerInfo's eq_derives list for + RestrictInfos that mention this relation */ +} EquivalenceClassIndexes; + /* * PathKeys * diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 8f86dd864e..565dce4040 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -663,6 +663,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceChildMemberIterator EquivalenceClass +EquivalenceClassIndexes EquivalenceMember ErrorContextCallback ErrorData -- 2.42.0.windows.2 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-03-06 14:16 Ashutosh Bapat <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Ashutosh Bapat @ 2024-03-06 14:16 UTC (permalink / raw) To: Yuya Watari <[email protected]>; +Cc: Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hi Yuya On Wed, Feb 28, 2024 at 4:48 PM Yuya Watari <[email protected]> wrote: > Hello, > > On Tue, Feb 13, 2024 at 6:19 AM Alena Rybakina <[email protected]> > wrote: > > > > Yes, it is working correctly now with the assertion check. I suppose > > it's better to add this code with an additional comment and a > > recommendation for other developers > > to use it for checking in case of manipulations with the list of > > equivalences. > > Thank you for your reply and advice. I have added this assertion so > that other developers can use it in the future. > > I also merged recent changes and attached a new version, v24. Since > this thread is getting long, I will summarize the patches. > > > I repeated my experiments in [1]. I ran 2, 3, 4, 5-way self-joins on a partitioned table with 1000 partitions. Planning time measurement --------------------------------------- Without patch with an assert enabled build and enable_partitionwise_join = false, those joins took 435.31 ms, 1629.16 ms, 4701.59 ms and 11976.69 ms respectively. Keeping other things the same, with the patch, they took 247.33 ms, 1318.57 ms, 6960.31 ms and 28463.24 ms respectively. Those with enable_partitionwise_join = true are 488.73 ms, 2102.12 ms, 6906.02 ms and 21300.77 ms respectively without the patch. And with the patch, 277.22 ms, 1542.48 ms, 7879.35 ms, and 31826.39 ms. Without patch without assert enabled build and enable_partitionwise_join = false, the joins take 298.43 ms, 1179.15 ms, 3518.84 ms and 9149.76 ms respectively. Keeping other things the same, with the patch, the joins take 65.70 ms, 131.29 ms, 247.67 ms and 477.74 ms respectively. Those with enable_partitionwise_join = true are 348.48 ms, 1576.11 ms, 5417.98 and 17433.65 ms respectively without the patch. And with the patch 95.15 ms, 333.99 ms, 1084.06 ms, and 3609.42 ms. Memory usage measurement --------------------------------------- Without patch, with an assert enabled build and enable_partitionwise_join = false, memory used is 19 MB, 45 MB, 83 MB and 149 MB respectively. Keeping other things the same, with the patch, memory used is 23 MB, 66 MB, 159 MB and 353 MB respectively. That with enable_partitionwise_join = true is 40 MB, 151 MB, 464 MB and 1663 MB respectively. And with the patch it is 44 MB, 172 MB, 540 MB and 1868 MB respectively. Without patch without assert enabled build and enable_partitionwise_join = false, memory used is 17 MB, 41 MB, 77 MB, and 140 MB resp. Keeping other things the same with the patch memory used is 21 MB, 62 MB, 152 MB and 341 MB resp. That with enable_partitionwise_join = true is 37 MB, 138 MB, 428 MB and 1495 MB resp. And with the patch it is 42 MB, 160 MB, 496 MB and 1705 MB resp. here's summary of observations 1. The patch improves planning time significantly (3X to 20X) and the improvement increases with the number of tables being joined. 2. In the assert enabled build the patch slows down (in comparison to HEAD) planning with higher number of tables in the join. You may want to investigate this. But this is still better than my earlier measurements. 3. The patch increased memory consumption by planner. But the numbers have improved since my last measurement. Still it will be good to investigate what causes this extra memory consumption. 4. Generally with the assert enabled build planner consumes more memory with or without patch. From my previous experience this might be due to Bitmapset objects created within Assert() calls. Does v24-0002 have any relation/overlap with my patches to reduce memory consumed by RestrictInfos? Those patches have code to avoid creating duplicate RestrictInfos (including commuted RestrictInfos) from ECs. [2] [1] https://www.postgresql.org/message-id/[email protected]... [2] https://www.postgresql.org/message-id/CAExHW5tEvzM%3D%2BLpN%3DyhU%2BP33D%2B%3D7x6fhzwDwNRM971UJunRTk... -- Best Wishes, Ashutosh Bapat ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-05-02 07:56 Yuya Watari <[email protected]> parent: Ashutosh Bapat <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-05-02 07:56 UTC (permalink / raw) To: Ashutosh Bapat <[email protected]>; +Cc: Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hello Ashutosh, Thank you for your email and for reviewing the patch. I sincerely apologize for the delay in responding. On Wed, Mar 6, 2024 at 11:16 PM Ashutosh Bapat <[email protected]> wrote: > here's summary of observations > 1. The patch improves planning time significantly (3X to 20X) and the improvement increases with the number of tables being joined. > 2. In the assert enabled build the patch slows down (in comparison to HEAD) planning with higher number of tables in the join. You may want to investigate this. But this is still better than my earlier measurements. > 3. The patch increased memory consumption by planner. But the numbers have improved since my last measurement. Still it will be good to investigate what causes this extra memory consumption. > 4. Generally with the assert enabled build planner consumes more memory with or without patch. From my previous experience this might be due to Bitmapset objects created within Assert() calls. Thank you for testing the patch and sharing the results. For comment #1, these results show the effectiveness of the patch. For comment #2, I agree that we should not slow down assert-enabled builds. The patch adds a lot of assertions to avoid adding bugs, but they might be too excessive. I will reconsider these assertions and remove unnecessary ones. For comments #3 and #4, while the patch improves time complexity, it has some negative impacts on space complexity. The patch uses a Bitmapset-based index to speed up searching for EquivalenceMembers and RestrictInfos. Reducing this memory consumption is a little hard, but this is a very important problem in committing this patch, so I will investigate this further. > Does v24-0002 have any relation/overlap with my patches to reduce memory consumed by RestrictInfos? Those patches have code to avoid creating duplicate RestrictInfos (including commuted RestrictInfos) from ECs. [2] Thank you for sharing these patches. My patch may be related to your patches. My patch speeds up slow linear searches over EquivalenceMembers and RestrictInfos. It uses several approaches, one of which is the Bitmapset-based index. Bitmapsets are space inefficient, so if there are many EquivalenceMembers and RestrictInfos, this index becomes large. This is true for highly partitioned cases, where there are a lot of similar (or duplicate) elements. Eliminating such duplicate elements may help my patch reduce memory consumption. I will investigate this further. Unfortunately, I've been busy due to work, so I may not be able to respond soon. I really apologize for this. However, I will look into the patches, including yours, and share further information if found. Again, I apologize for my late response and appreciate your kind review. -- Best regards, Yuya Watari ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-05-02 14:35 jian he <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: jian he @ 2024-05-02 14:35 UTC (permalink / raw) To: Yuya Watari <[email protected]>; +Cc: Ashutosh Bapat <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> On Thu, May 2, 2024 at 3:57 PM Yuya Watari <[email protected]> wrote: > hi. sorry to bother you, maybe a dumb question. trying to understand something under the hood. currently I only applied v24-0001-Speed-up-searches-for-child-EquivalenceMembers.patch. on v24-0001: +/* + * add_eq_member - build a new EquivalenceMember and add it to an EC + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + + ec->ec_members = lappend(ec->ec_members, em); + return em; +} + this part seems so weird to me. add_eq_member function was added very very long ago, why do we create a function with the same function name? also I didn't see deletion of original add_eq_member function (https://git.postgresql.org/cgit/postgresql.git/tree/src/backend/optimizer/path/equivclass.c#n516) in v24-0001. Obviously, now I cannot compile it correctly. What am I missing? ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-05-16 02:44 Yuya Watari <[email protected]> parent: jian he <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-05-16 02:44 UTC (permalink / raw) To: jian he <[email protected]>; +Cc: Ashutosh Bapat <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Developers <[email protected]> Hello, Thank you for reviewing these patches. On Thu, May 2, 2024 at 11:35 PM jian he <[email protected]> wrote: > on v24-0001: > +/* > + * add_eq_member - build a new EquivalenceMember and add it to an EC > + */ > +static EquivalenceMember * > +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, > + JoinDomain *jdomain, Oid datatype) > +{ > + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, > + NULL, datatype); > + > + ec->ec_members = lappend(ec->ec_members, em); > + return em; > +} > + > this part seems so weird to me. > add_eq_member function was added very very long ago, > why do we create a function with the same function name? > > also I didn't see deletion of original add_eq_member function > (https://git.postgresql.org/cgit/postgresql.git/tree/src/backend/optimizer/path/equivclass.c#n516) > in v24-0001. Actually, this patch does not recreate the add_eq_member() function but splits it into two functions: add_eq_member() and make_eq_member(). The reason why planning takes so long time in the current implementation is that EquivalenceClasses have a large number of child EquivalenceMembers, and the linear search for them is time-consuming. To solve this problem, the patch makes EquivalenceClasses have only parent members. There are few parent members, so we can speed up the search. In the patch, the child members are introduced when needed. The add_eq_member() function originally created EquivalenceMembers and added them to ec_members. In the patch, this function is split into the following two functions. 1. make_eq_member Creates a new (parent or child) EquivalenceMember and returns it without adding it to ec_members. 2. add_eq_member Creates a new parent (not child) EquivalenceMember and adds it to ec_members. Internally calls make_eq_member. When we create parent members, we simply call add_eq_member(). This is the same as the current implementation. When we create child members, we have to do something different. Look at the code below. The add_child_rel_equivalences() function creates child members. The patch creates child EquivalenceMembers by the make_eq_member() function and stores them in RelOptInfo (child_rel->eclass_child_members) instead of their parent EquivalenceClass->ec_members. When we need child EquivalenceMembers, we get them via RelOptInfos. ===== void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, RelOptInfo *parent_rel, RelOptInfo *child_rel) { ... i = -1; while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { ... foreach(lc, cur_ec->ec_members) { ... if (bms_is_subset(cur_em->em_relids, top_parent_relids) && !bms_is_empty(cur_em->em_relids)) { /* OK, generate transformed child version */ ... child_em = make_eq_member(cur_ec, child_expr, new_relids, cur_em->em_jdomain, cur_em, cur_em->em_datatype); child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, child_em); ... } } } } ===== I didn't change the name of add_eq_member, but it might be better to change it to something like add_parent_eq_member(). Alternatively, creating a new function named add_child_eq_member() that adds child members to RelOptInfo can be a solution. I will consider these changes in the next version. > Obviously, now I cannot compile it correctly. > What am I missing? Thank you for pointing this out. This is due to a conflict with a recent commit [1]. This commit introduces a new function named add_setop_child_rel_equivalences(), which is quoted below. This function creates a new child EquivalenceMember by calling add_eq_member(). We have to adjust this function to make my patch work, but it is not so easy. I'm sorry it will take some time to solve this conflict, but I will post a new version when it is fixed. ===== /* * add_setop_child_rel_equivalences * Add equivalence members for each non-resjunk target in 'child_tlist' * to the EquivalenceClass in the corresponding setop_pathkey's pk_eclass. * * 'root' is the PlannerInfo belonging to the top-level set operation. * 'child_rel' is the RelOptInfo of the child relation we're adding * EquivalenceMembers for. * 'child_tlist' is the target list for the setop child relation. The target * list expressions are what we add as EquivalenceMembers. * 'setop_pathkeys' is a list of PathKeys which must contain an entry for each * non-resjunk target in 'child_tlist'. */ void add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, List *child_tlist, List *setop_pathkeys) { ListCell *lc; ListCell *lc2 = list_head(setop_pathkeys); foreach(lc, child_tlist) { TargetEntry *tle = lfirst_node(TargetEntry, lc); EquivalenceMember *parent_em; PathKey *pk; if (tle->resjunk) continue; if (lc2 == NULL) elog(ERROR, "too few pathkeys for set operation"); pk = lfirst_node(PathKey, lc2); parent_em = linitial(pk->pk_eclass->ec_members); /* * We can safely pass the parent member as the first member in the * ec_members list as this is added first in generate_union_paths, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ add_eq_member(pk->pk_eclass, tle->expr, child_rel->relids, parent_em->em_jdomain, parent_em, exprType((Node *) tle->expr)); lc2 = lnext(setop_pathkeys, lc2); } /* * transformSetOperationStmt() ensures that the targetlist never contains * any resjunk columns, so all eclasses that exist in 'root' must have * received a new member in the loop above. Add them to the child_rel's * eclass_indexes. */ child_rel->eclass_indexes = bms_add_range(child_rel->eclass_indexes, 0, list_length(root->eq_classes) - 1); } ===== [1] https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=66c0185a3d14bbbf51d0fc9d267093ffec735... -- Best regards, Yuya Watari ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-08-29 05:34 Yuya Watari <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-08-29 05:34 UTC (permalink / raw) To: PostgreSQL Developers <[email protected]>; +Cc: jian he <[email protected]>; Ashutosh Bapat <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> Hello, On Thu, May 16, 2024 at 11:44 AM Yuya Watari <[email protected]> wrote: > I'm sorry it will take some time to solve > this conflict, but I will post a new version when it is fixed. The previous patches no longer apply to the master, so I rebased them. I'm sorry for the delay. I will summarize the patches again. 1. v25-0001 This patch is one of the main parts of my optimization. Traditionally, EquivalenceClass has both parent and child members. However, this leads to high iteration costs when there are many child partitions. In v25-0001, EquivalenceClasses no longer have child members. If we need to iterate over child EquivalenceMembers, we use the EquivalenceChildMemberIterator and access the children through the iterator. For more details, see [1] (note that there are some design changes from [1]). 2. v25-0002 This patch was made in the previous work with David. Like EquivalenceClass, there are many RestrictInfos in highly partitioned cases. This patch introduces an indexing mechanism to speed up searches for RestrictInfos. 3. v25-0003 v25-0002 adds its indexes to RangeTblEntry, but this is not a good idea. RelOptInfo is the best place. This problem is a workaround because some RelOptInfos can be NULL, so we cannot store indexes to such RelOptInfos. v25-0003 moves the indexes from RangeTblEntry to PlannerInfo. This is still a workaround, and I think it should be reconsidered. 4. v25-0004 After our changes, add_eq_member() no longer creates and adds child EquivalenceMembers. This commit renames it to add_parent_eq_member() to clarify that it only creates parent members, and that we need to use make_eq_member() to handle child EquivalenceMembers. 5. v25-0005 This commit resolves a conflict with commit 66c0185 [2], which adds add_setop_child_rel_equivalences(). As I mentioned in the previous email [3], this function creates child EquivalenceMembers by calling add_eq_member(). This commit adjusts our optimization so that it can handle such child members. [1] https://www.postgresql.org/message-id/CAJ2pMkZk-Nr%3DyCKrGfGLu35gK-D179QPyxaqtJMUkO86y1NmSA%40mail.g... [2] https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=66c0185a3 [3] https://www.postgresql.org/message-id/CAJ2pMkZt6r94NUYm-F77FYahjgnMrY4CLHGAD7HxYZxGVwCaow%40mail.gma... -- Best regards, Yuya Watari Attachments: [application/octet-stream] v25-0001-Speed-up-searches-for-child-EquivalenceMembers.patch (58.8K, ../../CAJ2pMkaWUaM53zoTFaLb1M8TX-2xmH5xk7dzvfjo2hSvy_zg3g@mail.gmail.com/2-v25-0001-Speed-up-searches-for-child-EquivalenceMembers.patch) download | inline diff: From ec975ba236ca94a2b78b92ddd576c9c030203a49 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:43:36 +0900 Subject: [PATCH v25 1/5] Speed up searches for child EquivalenceMembers Traditionally, child EquivalenceMembers were in the EquivalenceClass->ec_members. When we wanted to find some child members matching a request, we had to perform a linear search. This search became heavy when tables had many partitions, leading to much planning time. After this commit, child EquivalenceMembers no longer exist in ec_members. Instead, RelOptInfos have them. This change demonstrates a significant performance improvement in planning time. --- contrib/postgres_fdw/postgres_fdw.c | 22 +- src/backend/optimizer/path/equivclass.c | 414 ++++++++++++++++++++---- src/backend/optimizer/path/indxpath.c | 35 +- src/backend/optimizer/path/pathkeys.c | 9 +- src/backend/optimizer/plan/createplan.c | 57 ++-- src/backend/optimizer/util/inherit.c | 14 + src/backend/optimizer/util/relnode.c | 89 +++++ src/include/nodes/pathnodes.h | 61 ++++ src/include/optimizer/pathnode.h | 18 ++ src/include/optimizer/paths.h | 12 +- src/tools/pgindent/typedefs.list | 1 + 11 files changed, 619 insertions(+), 113 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index adc62576d1..93cbc53f03 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7831,13 +7831,21 @@ conversion_error_callback(void *arg) EquivalenceMember * find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) { - ListCell *lc; - PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + Relids top_parent_rel_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, rel->relids); /* * Note we require !bms_is_empty, else we'd accept constant @@ -7849,6 +7857,7 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) is_foreign_expr(root, rel, em->em_expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -7902,9 +7911,8 @@ find_em_for_rel_target(PlannerInfo *root, EquivalenceClass *ec, if (em->em_is_const) continue; - /* Ignore child members */ - if (em->em_is_child) - continue; + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* Match if same expression (after stripping relabel) */ em_expr = em->em_expr; diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index d871396e20..5f96435891 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -33,10 +33,14 @@ #include "utils/lsyscache.h" +static EquivalenceMember *make_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + EquivalenceMember *parent, + Oid datatype); static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, - EquivalenceMember *parent, Oid datatype); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, @@ -69,6 +73,11 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo); static JoinDomain *find_join_domain(PlannerInfo *root, Relids relids); +static void add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel); static Bitmapset *get_eclass_indexes_for_relids(PlannerInfo *root, Relids relids); static Bitmapset *get_common_eclass_indexes(PlannerInfo *root, Relids relids1, @@ -374,7 +383,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); @@ -391,7 +400,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); @@ -423,9 +432,9 @@ process_equivalence(PlannerInfo *root, ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; em1 = add_eq_member(ec, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); em2 = add_eq_member(ec, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -511,11 +520,16 @@ canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation) } /* - * add_eq_member - build a new EquivalenceMember and add it to an EC + * make_eq_member + * + * Build a new EquivalenceMember without adding it to an EC. If 'parent' + * parameter is NULL, the result will be a parent member, otherwise a child + * member. Note that child EquivalenceMembers should not be added to its + * parent EquivalenceClass. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) +make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) { EquivalenceMember *em = makeNode(EquivalenceMember); @@ -526,6 +540,8 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, em->em_datatype = datatype; em->em_jdomain = jdomain; em->em_parent = parent; + em->em_child_relids = NULL; + em->em_child_joinrel_relids = NULL; if (bms_is_empty(relids)) { @@ -546,11 +562,29 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, { ec->ec_relids = bms_add_members(ec->ec_relids, relids); } - ec->ec_members = lappend(ec->ec_members, em); return em; } +/* + * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * + * Note: We don't have a function to add a child member like + * add_child_eq_member() because how to do it depends on the relations they are + * translated from. See add_child_rel_equivalences() and + * add_child_join_rel_equivalences() to see how to add child members. + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + + ec->ec_members = lappend(ec->ec_members, em); + return em; +} + /* * get_eclass_for_sort_expr @@ -599,6 +633,17 @@ get_eclass_for_sort_expr(PlannerInfo *root, EquivalenceMember *newem; ListCell *lc1; MemoryContext oldcontext; + Relids top_parent_rel; + + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_rel = find_relids_top_parents(root, rel); /* * Ensure the expression exposes the correct type and collation. @@ -617,7 +662,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; /* * Never match to a volatile EC, except when we are looking at another @@ -632,16 +678,28 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (!equal(opfamilies, cur_ec->ec_opfamilies)) continue; - foreach(lc2, cur_ec->ec_members) + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel); /* - * Ignore child members unless they match the request. + * Ignore child members unless they match the request. If this + * EquivalenceMember is a child, i.e., translated above, it should + * match the request. We cannot assert this if a request is + * bms_is_subset(). */ - if (cur_em->em_is_child && - !bms_equal(cur_em->em_relids, rel)) - continue; + Assert(!cur_em->em_is_child || bms_equal(cur_em->em_relids, rel)); /* * Match constants only within the same JoinDomain (see @@ -654,6 +712,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, equal(expr, cur_em->em_expr)) return cur_ec; /* Match! */ } + dispose_eclass_child_member_iterator(&it); } /* No match; does caller want a NULL result? */ @@ -691,7 +750,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, expr_relids = pull_varnos(root, (Node *) expr); newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, NULL, opcintype); + jdomain, opcintype); /* * add_eq_member doesn't check for volatile functions, set-returning @@ -757,19 +816,25 @@ get_eclass_for_sort_expr(PlannerInfo *root, * Child EC members are ignored unless they belong to given 'relids'. */ EquivalenceMember * -find_ec_member_matching_expr(EquivalenceClass *ec, +find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, Expr *expr, Relids relids) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); /* We ignore binary-compatible relabeling on both ends */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); Expr *emexpr; /* @@ -779,6 +844,11 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -796,6 +866,7 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (equal(emexpr, expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -828,11 +899,17 @@ find_computable_ec_member(PlannerInfo *root, Relids relids, bool require_parallel_safe) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); List *exprvars; ListCell *lc2; @@ -843,6 +920,11 @@ find_computable_ec_member(PlannerInfo *root, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -876,6 +958,7 @@ find_computable_ec_member(PlannerInfo *root, return em; /* found usable expression */ } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -939,7 +1022,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, { Expr *targetexpr = (Expr *) lfirst(lc); - em = find_ec_member_matching_expr(ec, targetexpr, rel->relids); + em = find_ec_member_matching_expr(root, ec, targetexpr, rel->relids); if (!em) continue; @@ -1559,7 +1642,12 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *new_members = NIL; List *outer_members = NIL; List *inner_members = NIL; - ListCell *lc1; + Relids top_parent_join_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_join_relids = find_relids_top_parents(root, join_relids); /* * First, scan the EC to identify member values that are computable at the @@ -1570,9 +1658,14 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * as well as to at least one input member, plus enforce at least one * outer-rel member equal to at least one inner-rel member. */ - foreach(lc1, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + bms_is_subset(cur_em->em_relids, top_parent_join_relids)) + iterate_child_rel_equivalences(&it, root, ec, cur_em, join_relids); /* * We don't need to check explicitly for child EC members. This test @@ -1589,6 +1682,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, else new_members = lappend(new_members, cur_em); } + dispose_eclass_child_member_iterator(&it); /* * First, select the joinclause if needed. We can equate any one outer @@ -1606,6 +1700,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, Oid best_eq_op = InvalidOid; int best_score = -1; RestrictInfo *rinfo; + ListCell *lc1; foreach(lc1, outer_members) { @@ -1680,6 +1775,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *old_members = list_concat(outer_members, inner_members); EquivalenceMember *prev_em = NULL; RestrictInfo *rinfo; + ListCell *lc1; /* For now, arbitrarily take the first old_member as the one to use */ if (old_members) @@ -1687,7 +1783,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, foreach(lc1, new_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + cur_em = (EquivalenceMember *) lfirst(lc1); if (prev_em != NULL) { @@ -2400,6 +2496,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) if (matchleft && matchright) { cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); + Assert(!bms_is_empty(coal_em->em_relids)); return true; } @@ -2483,8 +2580,8 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, Oid opfamily) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); if (equal(item1, em->em_expr)) item1member = true; else if (equal(item2, em->em_expr)) @@ -2555,8 +2652,8 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); Var *var; - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* EM must be a Var, possibly with RelabelType */ var = (Var *) em->em_expr; @@ -2653,6 +2750,7 @@ add_child_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_rel->top_parent_relids; Relids child_relids = child_rel->relids; int i; + ListCell *lc; /* * EC merging should be complete already, so we can use the parent rel's @@ -2665,7 +2763,6 @@ add_child_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2678,15 +2775,9 @@ add_child_rel_equivalences(PlannerInfo *root, /* Sanity check eclass_indexes only contain ECs for parent_rel */ Assert(bms_is_subset(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2699,8 +2790,8 @@ add_child_rel_equivalences(PlannerInfo *root, * combinations of children. (But add_child_join_rel_equivalences * may add targeted combinations for partitionwise-join purposes.) */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * Consider only members that reference and can be computed at @@ -2716,6 +2807,7 @@ add_child_rel_equivalences(PlannerInfo *root, /* OK, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_rel->reloptkind == RELOPT_BASEREL) { @@ -2745,9 +2837,20 @@ add_child_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, + child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_relids = bms_add_member(cur_em->em_child_relids, + child_rel->relid); /* Record this EC index for the child rel */ child_rel->eclass_indexes = bms_add_member(child_rel->eclass_indexes, i); @@ -2776,6 +2879,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; MemoryContext oldcontext; + ListCell *lc; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2797,7 +2901,6 @@ add_child_join_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(matching_ecs, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2810,15 +2913,9 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Sanity check on get_eclass_indexes_for_relids result */ Assert(bms_overlap(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2827,8 +2924,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, * We consider only original EC members here, not * already-transformed child members. */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * We may ignore expressions that reference a single baserel, @@ -2843,6 +2940,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Yes, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_joinrel->reloptkind == RELOPT_JOINREL) { @@ -2873,9 +2971,21 @@ add_child_join_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_joinrel->eclass_child_members = + lappend(child_joinrel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_joinrel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_joinrel_relids = + bms_add_member(cur_em->em_child_joinrel_relids, + child_joinrel->join_rel_list_index); } } } @@ -2944,6 +3054,161 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, list_length(root->eq_classes) - 1); } +/* + * setup_eclass_child_member_iterator + * Setup an EquivalenceChildMemberIterator 'it' so that it can iterate over + * EquivalenceMembers in 'ec'. + * + * Once used, the caller should dispose of the iterator by calling + * dispose_eclass_child_member_iterator(). + */ +void +setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec) +{ + it->index = -1; + it->modified = false; + it->ec_members = ec->ec_members; +} + +/* + * eclass_child_member_iterator_next + * Get a next EquivalenceMember from an EquivalenceChildMemberIterator 'it' + * that was setup by setup_eclass_child_member_iterator(). NULL is returned + * if there are no members left. + */ +EquivalenceMember * +eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it) +{ + if (++it->index < list_length(it->ec_members)) + return list_nth_node(EquivalenceMember, it->ec_members, it->index); + return NULL; +} + +/* + * dispose_eclass_child_member_iterator + * Free any memory allocated by the iterator. + */ +void +dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it) +{ + /* + * XXX Should we use list_free()? I decided to use this style to take + * advantage of speculative execution. + */ + if (unlikely(it->modified)) + pfree(it->ec_members); +} + +/* + * add_transformed_child_version + * Add a transformed EquivalenceMember referencing the given child_rel to + * the iterator. + * + * This function is expected to be called only from + * iterate_child_rel_equivalences(). + */ +static void +add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel) +{ + ListCell *lc; + + foreach(lc, child_rel->eclass_child_members) + { + EquivalenceMember *child_em = lfirst_node(EquivalenceMember, lc); + + /* Skip unwanted EquivalenceMembers */ + if (child_em->em_parent != parent_em) + continue; + + /* + * If this is the first time the iterator's list has been modified, we + * need to make a copy of it. + */ + if (!it->modified) + { + it->ec_members = list_copy(it->ec_members); + it->modified = true; + } + + /* Add this child EquivalenceMember to the list */ + it->ec_members = lappend(it->ec_members, child_em); + } +} + +/* + * iterate_child_rel_equivalences + * Add transformed EquivalenceMembers referencing child rels in the given + * child_relids to the iterator. + * + * The transformation is done in add_transformed_child_version(). + */ +void +iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids) +{ + int i; + Relids matching_relids; + + /* The given EquivalenceMember should be parent */ + Assert(!parent_em->em_is_child); + + /* + * EquivalenceClass->ec_members has only parent EquivalenceMembers. Child + * members are translated using child RelOptInfos and stored in them. This + * is done in add_child_rel_equivalences() and + * add_child_join_rel_equivalences(). To retrieve child EquivalenceMembers + * of some parent, we need to know which RelOptInfos have such child + * members. This information is stored in indexes named em_child_relids + * and em_child_joinrel_relids. + * + * This function iterates over the child EquivalenceMembers that reference + * the given 'child_relids'. To do this, we first intersect 'child_relids' + * with these indexes. The result contains Relids of RelOptInfos that have + * child EquivalenceMembers we want to retrieve. Then we get the child + * members from the RelOptInfos using add_transformed_child_version(). + * + * We need to do these steps for each of the two indexes. + */ + + /* + * First, we translate simple rels. + */ + matching_relids = bms_intersect(parent_em->em_child_relids, + child_relids); + i = -1; + while ((i = bms_next_member(matching_relids, i)) >= 0) + { + /* Add transformed child version for this child rel */ + RelOptInfo *child_rel = root->simple_rel_array[i]; + + Assert(child_rel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_rel); + } + bms_free(matching_relids); + + /* + * Next, we try to translate join rels. + */ + i = -1; + while ((i = bms_next_member(parent_em->em_child_joinrel_relids, i)) >= 0) + { + /* Add transformed child version for this child join rel */ + RelOptInfo *child_joinrel = + list_nth_node(RelOptInfo, root->join_rel_list, i); + + Assert(child_joinrel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_joinrel); + } +} + /* * generate_implied_equalities_for_column @@ -2977,7 +3242,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, { List *result = NIL; bool is_child_rel = (rel->reloptkind == RELOPT_OTHER_MEMBER_REL); - Relids parent_relids; + Relids parent_relids, + top_parent_rel_relids; int i; /* Should be OK to rely on eclass_indexes */ @@ -2986,6 +3252,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, /* Indexes are available only on base or "other" member relations. */ Assert(IS_SIMPLE_REL(rel)); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + /* If it's a child rel, we'll need to know what its parent(s) are */ if (is_child_rel) parent_relids = find_childrel_parents(root, rel); @@ -2998,6 +3267,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); EquivalenceMember *cur_em; ListCell *lc2; + EquivalenceChildMemberIterator it; /* Sanity check eclass_indexes only contain ECs for rel */ Assert(is_child_rel || bms_is_subset(rel->relids, cur_ec->ec_relids)); @@ -3019,15 +3289,19 @@ generate_implied_equalities_for_column(PlannerInfo *root, * corner cases, so for now we live with just reporting the first * match. See also get_eclass_for_sort_expr.) */ - cur_em = NULL; - foreach(lc2, cur_ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - cur_em = (EquivalenceMember *) lfirst(lc2); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel->relids); if (bms_equal(cur_em->em_relids, rel->relids) && callback(root, rel, cur_ec, cur_em, callback_arg)) break; - cur_em = NULL; } + dispose_eclass_child_member_iterator(&it); if (!cur_em) continue; @@ -3042,8 +3316,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, Oid eq_op; RestrictInfo *rinfo; - if (other_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!other_em->em_is_child); /* Make sure it'll be a join to a different rel */ if (other_em == cur_em || @@ -3261,8 +3535,8 @@ eclass_useful_for_merging(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); if (!bms_overlap(cur_em->em_relids, relids)) return true; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index c0fcc7d78d..ce14ca324f 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -183,7 +183,7 @@ static IndexClause *expand_indexqual_rowcompare(PlannerInfo *root, IndexOptInfo *index, Oid expr_op, bool var_on_left); -static void match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +static void match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p); static Expr *match_clause_to_ordering_op(IndexOptInfo *index, @@ -927,7 +927,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * query_pathkeys will allow an incremental sort to be considered on * the index's partially sorted results. */ - match_pathkeys_to_index(index, root->query_pathkeys, + match_pathkeys_to_index(root, index, root->query_pathkeys, &orderbyclauses, &orderbyclausecols); if (list_length(root->query_pathkeys) == list_length(orderbyclauses)) @@ -3017,12 +3017,16 @@ expand_indexqual_rowcompare(PlannerInfo *root, * item in the given 'pathkeys' list. */ static void -match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p) { + Relids top_parent_index_relids; ListCell *lc1; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_index_relids = find_relids_top_parents(root, index->rel->relids); + *orderby_clauses_p = NIL; /* set default results */ *clause_columns_p = NIL; @@ -3034,7 +3038,8 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, { PathKey *pathkey = (PathKey *) lfirst(lc1); bool found = false; - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *member; /* Pathkey must request default sort order for the target opfamily */ @@ -3054,15 +3059,30 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, * be considered to match more than one pathkey list, which is OK * here. See also get_eclass_for_sort_expr.) */ - foreach(lc2, pathkey->pk_eclass->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, pathkey->pk_eclass); + while ((member = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *member = (EquivalenceMember *) lfirst(lc2); int indexcol; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + bms_equal(member->em_relids, top_parent_index_relids)) + iterate_child_rel_equivalences(&it, root, pathkey->pk_eclass, member, + index->rel->relids); + /* No possibility of match if it references other relations */ - if (!bms_equal(member->em_relids, index->rel->relids)) + if (!member->em_is_child && + !bms_equal(member->em_relids, index->rel->relids)) continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(bms_equal(member->em_relids, index->rel->relids)); + /* * We allow any column of the index to match each pathkey; they * don't have to match left-to-right as you might expect. This is @@ -3091,6 +3111,7 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, if (found) /* don't want to look at remaining members */ break; } + dispose_eclass_child_member_iterator(&it); /* * Return the matches found so far when this pathkey couldn't be diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index e25798972f..c89e415c34 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -1149,8 +1149,8 @@ convert_subquery_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid sub_expr_coll = sub_eclass->ec_collation; ListCell *k; - if (sub_member->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!sub_member->em_is_child); foreach(k, subquery_tlist) { @@ -1693,8 +1693,11 @@ select_outer_pathkeys_for_merge(PlannerInfo *root, { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); + /* Potential future join partner? */ - if (!em->em_is_const && !em->em_is_child && + if (!em->em_is_const && !bms_overlap(em->em_relids, joinrel->relids)) score++; } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 8e0e5977a9..74ee5cb845 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -262,7 +262,9 @@ static IncrementalSort *make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); -static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -271,9 +273,11 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); -static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Sort *make_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids); -static IncrementalSort *make_incrementalsort_from_pathkeys(Plan *lefttree, +static IncrementalSort *make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, int nPresortedCols); static Sort *make_sort_from_groupcols(List *groupcls, AttrNumber *grpColIdx, @@ -295,7 +299,7 @@ static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); static Unique *make_unique_from_sortclauses(Plan *lefttree, List *distinctList); -static Unique *make_unique_from_pathkeys(Plan *lefttree, +static Unique *make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, int numCols); static Gather *make_gather(List *qptlist, List *qpqual, int nworkers, int rescan_param, bool single_copy, Plan *subplan); @@ -1283,7 +1287,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * function result; it must be the same plan node. However, we then * need to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys((Plan *) plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, (Plan *) plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1327,7 +1331,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * don't need an explicit sort, to make sure they are returning * the same sort key columns the Append expects. */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, nodeSortColIdx, false, @@ -1468,7 +1472,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * function result; it must be the same plan node. However, we then need * to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys(plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1499,7 +1503,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, subplan = create_plan_recurse(root, subpath, CP_EXACT_TLIST); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, node->sortColIdx, false, @@ -1978,7 +1982,7 @@ create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) Assert(pathkeys != NIL); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, best_path->subpath->parent->relids, gm_plan->sortColIdx, false, @@ -2192,7 +2196,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) * relids. Thus, if this sort path is based on a child relation, we must * pass its relids. */ - plan = make_sort_from_pathkeys(subplan, best_path->path.pathkeys, + plan = make_sort_from_pathkeys(root, subplan, best_path->path.pathkeys, IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -2216,7 +2220,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, /* See comments in create_sort_plan() above */ subplan = create_plan_recurse(root, best_path->spath.subpath, flags | CP_SMALL_TLIST); - plan = make_incrementalsort_from_pathkeys(subplan, + plan = make_incrementalsort_from_pathkeys(root, subplan, best_path->spath.path.pathkeys, IS_OTHER_REL(best_path->spath.subpath->parent) ? best_path->spath.path.parent->relids : NULL, @@ -2285,7 +2289,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag subplan = create_plan_recurse(root, best_path->subpath, flags | CP_LABEL_TLIST); - plan = make_unique_from_pathkeys(subplan, + plan = make_unique_from_pathkeys(root, subplan, best_path->path.pathkeys, best_path->numkeys); @@ -4523,7 +4527,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->outersortkeys) { Relids outer_relids = outer_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(outer_plan, + Sort *sort = make_sort_from_pathkeys(root, outer_plan, best_path->outersortkeys, outer_relids); @@ -4537,7 +4541,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->innersortkeys) { Relids inner_relids = inner_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(inner_plan, + Sort *sort = make_sort_from_pathkeys(root, inner_plan, best_path->innersortkeys, inner_relids); @@ -6161,7 +6165,7 @@ make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, * or a Result stacked atop lefttree). */ static Plan * -prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -6228,7 +6232,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ @@ -6256,7 +6260,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr already in tlist */ @@ -6272,7 +6276,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, /* * No matching tlist item; look for a computable expression. */ - em = find_computable_ec_member(NULL, ec, tlist, relids, false); + em = find_computable_ec_member(root, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; @@ -6343,7 +6347,8 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ static Sort * -make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) +make_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + Relids relids) { int numsortkeys; AttrNumber *sortColIdx; @@ -6352,7 +6357,7 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6378,8 +6383,9 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) * 'nPresortedCols' is the number of presorted columns in input tuples */ static IncrementalSort * -make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, - Relids relids, int nPresortedCols) +make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, + List *pathkeys, Relids relids, + int nPresortedCols) { int numsortkeys; AttrNumber *sortColIdx; @@ -6388,7 +6394,7 @@ make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6747,7 +6753,8 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) * as above, but use pathkeys to identify the sort columns and semantics */ static Unique * -make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) +make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + int numCols) { Unique *node = makeNode(Unique); Plan *plan = &node->plan; @@ -6810,7 +6817,7 @@ make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) foreach(j, plan->targetlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, NULL); + em = find_ec_member_matching_expr(root, ec, tle->expr, NULL); if (em) { /* found expr already in tlist */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index c5b906a9d4..3c2198ea5b 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -470,6 +470,7 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, RelationGetRelid(parentrel); Oid childOID = RelationGetRelid(childrel); RangeTblEntry *childrte; + Index topParentRTindex; Index childRTindex; AppendRelInfo *appinfo; TupleDesc child_tupdesc; @@ -577,6 +578,19 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, Assert(root->append_rel_array[childRTindex] == NULL); root->append_rel_array[childRTindex] = appinfo; + /* + * Find a top parent rel's index and save it to top_parent_relid_array. + */ + if (root->top_parent_relid_array == NULL) + root->top_parent_relid_array = + (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); + Assert(root->top_parent_relid_array[childRTindex] == 0); + topParentRTindex = parentRTindex; + while (root->append_rel_array[topParentRTindex] != NULL && + root->append_rel_array[topParentRTindex]->parent_relid != 0) + topParentRTindex = root->append_rel_array[topParentRTindex]->parent_relid; + root->top_parent_relid_array[childRTindex] = topParentRTindex; + /* * Build a PlanRowMark if parent is marked FOR UPDATE/SHARE. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index d7266e4cdb..f32575b949 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -123,11 +123,14 @@ setup_simple_rel_arrays(PlannerInfo *root) if (root->append_rel_list == NIL) { root->append_rel_array = NULL; + root->top_parent_relid_array = NULL; return; } root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); + root->top_parent_relid_array = (Index *) + palloc0(size * sizeof(Index)); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -148,6 +151,27 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array[child_relid] = appinfo; } + + /* + * Find a top parent rel's relid for each AppendRelInfo. We cannot do this + * in the last foreach loop because there may be multi-level parent-child + * relations. + */ + for (int i = 0; i < size; i++) + { + int top_parent_relid; + + if (root->append_rel_array[i] == NULL) + continue; + + top_parent_relid = root->append_rel_array[i]->parent_relid; + + while (root->append_rel_array[top_parent_relid] != NULL && + root->append_rel_array[top_parent_relid]->parent_relid != 0) + top_parent_relid = root->append_rel_array[top_parent_relid]->parent_relid; + + root->top_parent_relid_array[i] = top_parent_relid; + } } /* @@ -175,12 +199,25 @@ expand_planner_arrays(PlannerInfo *root, int add_size) repalloc0_array(root->simple_rte_array, RangeTblEntry *, root->simple_rel_array_size, new_size); if (root->append_rel_array) + { root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); + root->top_parent_relid_array = + repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + } else + { root->append_rel_array = palloc0_array(AppendRelInfo *, new_size); + /* + * We do not allocate top_parent_relid_array here so that + * find_relids_top_parents() can early find all of the given Relids + * are top-level. + */ + root->top_parent_relid_array = NULL; + } + root->simple_rel_array_size = new_size; } @@ -234,6 +271,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; + rel->eclass_child_members = NIL; rel->serverid = InvalidOid; if (rte->rtekind == RTE_RELATION) { @@ -629,6 +667,12 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) /* GEQO requires us to append the new joinrel to the end of the list! */ root->join_rel_list = lappend(root->join_rel_list, joinrel); + /* + * Store the index of this joinrel to use in + * add_child_join_rel_equivalences(). + */ + joinrel->join_rel_list_index = list_length(root->join_rel_list) - 1; + /* store it into the auxiliary hashtable if there is one. */ if (root->join_rel_hash) { @@ -741,6 +785,7 @@ build_join_rel(PlannerInfo *root, joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -928,6 +973,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -1490,6 +1536,7 @@ fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids) upperrel->cheapest_total_path = NULL; upperrel->cheapest_unique_path = NULL; upperrel->cheapest_parameterized_paths = NIL; + upperrel->eclass_child_members = NIL; /* XXX Is this required? */ root->upper_rels[kind] = lappend(root->upper_rels[kind], upperrel); @@ -1530,6 +1577,48 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } +/* + * find_relids_top_parents_slow + * Slow path of find_relids_top_parents() macro. + * + * Do not call this directly; use the macro instead. See the macro comment for + * more details. + */ +Relids +find_relids_top_parents_slow(PlannerInfo *root, Relids relids) +{ + Index *top_parent_relid_array = root->top_parent_relid_array; + Relids result; + bool is_top_parent; + int i; + + /* This function should be called in the slow path */ + Assert(top_parent_relid_array != NULL); + + result = NULL; + is_top_parent = true; + i = -1; + while ((i = bms_next_member(relids, i)) >= 0) + { + int top_parent_relid = (int) top_parent_relid_array[i]; + + if (top_parent_relid == 0) + top_parent_relid = i; /* 'i' has no parents, so add itself */ + else if (top_parent_relid != i) + is_top_parent = false; + result = bms_add_member(result, top_parent_relid); + } + + if (is_top_parent) + { + bms_free(result); + return NULL; + } + + return result; +} + + /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 540d021592..b625694e81 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -248,6 +248,14 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * top_parent_relid_array is the same length as simple_rel_array and holds + * the top-level parent indexes of the corresponding rels within + * simple_rel_array. The element can be zero if the rel has no parents, + * i.e., is itself in a top-level. + */ + Index *top_parent_relid_array pg_node_attr(read_write_ignore); + /* * all_baserels is a Relids set of all base relids (but not joins or * "other" rels) in the query. This is computed in deconstruct_jointree. @@ -951,6 +959,17 @@ typedef struct RelOptInfo /* Bitmask of optional features supported by the table AM */ uint32 amflags; + /* + * information about a join rel + */ + /* index in root->join_rel_list of this rel */ + Index join_rel_list_index; + + /* + * EquivalenceMembers referencing this rel + */ + List *eclass_child_members; + /* * Information about foreign tables and foreign joins */ @@ -1365,6 +1384,12 @@ typedef struct JoinDomain * entry: consider SELECT random() AS a, random() AS b ... ORDER BY b,a. * So we record the SortGroupRef of the originating sort clause. * + * EquivalenceClass->ec_members can only have parent members, and child members + * are stored in RelOptInfos, from which those child members are translated. To + * lookup child EquivalenceMembers, we use EquivalenceChildMemberIterator. See + * its comment for usage. The approach to lookup child members quickly is + * described as iterate_child_rel_equivalences() comment. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1437,10 +1462,46 @@ typedef struct EquivalenceMember bool em_is_child; /* derived version for a child relation? */ Oid em_datatype; /* the "nominal type" used by the opfamily */ JoinDomain *em_jdomain; /* join domain containing the source clause */ + Relids em_child_relids; /* all relids of child rels */ + Bitmapset *em_child_joinrel_relids; /* indexes in root->join_rel_list + * of join rel children */ /* if em_is_child is true, this links to corresponding EM for top parent */ struct EquivalenceMember *em_parent pg_node_attr(read_write_ignore); } EquivalenceMember; +/* + * EquivalenceChildMemberIterator + * + * EquivalenceClass has only parent EquivalenceMembers, so we need to translate + * child members if necessary. EquivalenceChildMemberIterator provides a way to + * iterate over translated child members during the loop in addition to all of + * the parent EquivalenceMembers. + * + * The most common way to use this struct is as follows: + * ----- + * EquivalenceClass *ec = given; + * Relids rel = given; + * EquivalenceChildMemberIterator it; + * EquivalenceMember *em; + * + * setup_eclass_child_member_iterator(&it, ec); + * while ((em = eclass_child_member_iterator_next(&it)) != NULL) + * { + * use em ...; + * if (we need to iterate over child EquivalenceMembers) + * iterate_child_rel_equivalences(&it, root, ec, em, rel); + * use em ...; + * } + * dispose_eclass_child_member_iterator(&it); + * ----- + */ +typedef struct +{ + int index; /* current index within 'ec_members'. */ + bool modified; /* is 'ec_members' a newly allocated one? */ + List *ec_members; /* parent and child members */ +} EquivalenceChildMemberIterator; + /* * PathKeys * diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 1035e6560c..5e79cf1f63 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -332,6 +332,24 @@ extern Relids min_join_parameterization(PlannerInfo *root, extern RelOptInfo *fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids); extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); + +/* + * find_relids_top_parents + * Compute the set of top-parent relids of rel. + * + * Replaces all Relids appearing in the given 'relids' as their top-level + * parents. The result will be NULL if and only if all of the given relids are + * top-level. + * + * The motivation for having this feature as a macro rather than a function is + * that Relids are top-level in most cases. We can quickly determine when + * root->top_parent_relid_array is NULL. + */ +#define find_relids_top_parents(root, relids) \ + (likely((root)->top_parent_relid_array == NULL) \ + ? NULL : find_relids_top_parents_slow(root, relids)) +extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); + extern ParamPathInfo *get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, Relids required_outer); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 970499c469..3894ef0644 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -136,7 +136,8 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Index sortref, Relids rel, bool create_it); -extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec, +extern EquivalenceMember *find_ec_member_matching_expr(PlannerInfo *root, + EquivalenceClass *ec, Expr *expr, Relids relids); extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root, @@ -178,6 +179,15 @@ extern void add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, List *child_tlist, List *setop_pathkeys); +extern void setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec); +extern EquivalenceMember *eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it); +extern void dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it); +extern void iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids); extern List *generate_implied_equalities_for_column(PlannerInfo *root, RelOptInfo *rel, ec_matches_callback_type callback, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 9e951a9e6f..bdd6b7fdd9 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -682,6 +682,7 @@ EphemeralNamedRelation EphemeralNamedRelationData EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData +EquivalenceChildMemberIterator EquivalenceClass EquivalenceMember ErrorContextCallback -- 2.45.2.windows.1 [application/octet-stream] v25-0002-Introduce-indexes-for-RestrictInfo.patch (42.9K, ../../CAJ2pMkaWUaM53zoTFaLb1M8TX-2xmH5xk7dzvfjo2hSvy_zg3g@mail.gmail.com/3-v25-0002-Introduce-indexes-for-RestrictInfo.patch) download | inline diff: From ac7396be79d595eff0d6f7a46051b3adca4b9b6b Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:44:25 +0900 Subject: [PATCH v25 2/5] Introduce indexes for RestrictInfo This commit adds indexes to speed up searches for RestrictInfos. When there are many child partitions and we have to perform planning for join operations on the tables, we have to handle a large number of RestrictInfos, resulting in a long planning time. This commit adds ec_[source|derive]_indexes to speed up the search. We can use the indexes to filter out unwanted RestrictInfos, and improve the planning performance. Author: David Rowley <[email protected]> and me, and includes rebase by Alena Rybakina <[email protected]> [1]. [1] https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/nodes/outfuncs.c | 6 +- src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/path/equivclass.c | 516 ++++++++++++++++++++-- src/backend/optimizer/plan/analyzejoins.c | 49 +- src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/appendinfo.c | 2 + src/backend/optimizer/util/inherit.c | 7 + src/backend/optimizer/util/restrictinfo.c | 5 + src/include/nodes/parsenodes.h | 6 + src/include/nodes/pathnodes.h | 41 +- src/include/optimizer/paths.h | 21 +- 13 files changed, 593 insertions(+), 69 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 3337b77ae6..1b3cae4fca 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -466,8 +466,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_opfamilies); WRITE_OID_FIELD(ec_collation); WRITE_NODE_FIELD(ec_members); - WRITE_NODE_FIELD(ec_sources); - WRITE_NODE_FIELD(ec_derives); + WRITE_BITMAPSET_FIELD(ec_source_indexes); + WRITE_BITMAPSET_FIELD(ec_derive_indexes); WRITE_BITMAPSET_FIELD(ec_relids); WRITE_BOOL_FIELD(ec_has_const); WRITE_BOOL_FIELD(ec_has_volatile); @@ -570,6 +570,8 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); + WRITE_BITMAPSET_FIELD(eclass_source_indexes); + WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index b47950764a..ccf06860fe 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -431,6 +431,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); + READ_BITMAPSET_FIELD(eclass_source_indexes); + READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index e1523d15df..f8c04ea6b0 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5792,7 +5792,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, if (ec && ec->ec_has_const) { EquivalenceMember *em = fkinfo->fk_eclass_member[i]; - RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + RestrictInfo *rinfo = find_derived_clause_for_ec_member(root, + ec, em); if (rinfo) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 5f96435891..f7d04c8f46 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -42,6 +42,10 @@ static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, Oid datatype); +static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); +static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); @@ -320,7 +324,6 @@ process_equivalence(PlannerInfo *root, /* If case 1, nothing to do, except add to sources */ if (ec1 == ec2) { - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -331,6 +334,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); return true; } @@ -351,8 +356,10 @@ process_equivalence(PlannerInfo *root, * be found. */ ec1->ec_members = list_concat(ec1->ec_members, ec2->ec_members); - ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); - ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); + ec1->ec_source_indexes = bms_join(ec1->ec_source_indexes, + ec2->ec_source_indexes); + ec1->ec_derive_indexes = bms_join(ec1->ec_derive_indexes, + ec2->ec_derive_indexes); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); ec1->ec_has_const |= ec2->ec_has_const; /* can't need to set has_volatile */ @@ -364,10 +371,9 @@ process_equivalence(PlannerInfo *root, root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; - ec2->ec_sources = NIL; - ec2->ec_derives = NIL; + ec2->ec_source_indexes = NULL; + ec2->ec_derive_indexes = NULL; ec2->ec_relids = NULL; - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -378,13 +384,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec1) { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, jdomain, item2_type); - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -395,13 +402,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec2) { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, jdomain, item1_type); - ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -412,6 +420,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec2, restrictinfo); } else { @@ -421,8 +431,8 @@ process_equivalence(PlannerInfo *root, ec->ec_opfamilies = opfamilies; ec->ec_collation = collation; ec->ec_members = NIL; - ec->ec_sources = list_make1(restrictinfo); - ec->ec_derives = NIL; + ec->ec_source_indexes = NULL; + ec->ec_derive_indexes = NULL; ec->ec_relids = NULL; ec->ec_has_const = false; ec->ec_has_volatile = false; @@ -444,6 +454,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec, restrictinfo); } return true; @@ -585,6 +597,167 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, return em; } +/* + * add_eq_source - add 'rinfo' in eq_sources for this 'ec' + */ +static void +add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int source_idx = list_length(root->eq_sources); + int i; + + ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); + root->eq_sources = lappend(root->eq_sources, rinfo); + Assert(rinfo->eq_sources_index == -1); + rinfo->eq_sources_index = source_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + source_idx); + } +} + +/* + * add_eq_derive - add 'rinfo' in eq_derives for this 'ec' + */ +static void +add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int derive_idx = list_length(root->eq_derives); + int i; + + ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); + root->eq_derives = lappend(root->eq_derives, rinfo); + Assert(rinfo->eq_derives_index == -1); + rinfo->eq_derives_index = derive_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + derive_idx); + } +} + +/* + * update_clause_relids + * Update RestrictInfo->clause_relids with adjusting the relevant indexes. + * The clause_relids field must be updated through this function, not by + * setting the field directly. + */ +void +update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids) +{ + int i; + Relids removing_relids; + Relids adding_relids; +#ifdef USE_ASSERT_CHECKING + Relids common_relids; +#endif + + /* + * If there is nothing to do, we return immediately after setting the + * clause_relids field. + */ + if (rinfo->eq_sources_index == -1 && rinfo->eq_derives_index == -1) + { + rinfo->clause_relids = new_clause_relids; + return; + } + + /* + * Remove any references between this RestrictInfo and RangeTblEntries + * that are no longer relevant. + */ + removing_relids = bms_difference(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(removing_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(removing_relids); + + /* + * Add references between this RestrictInfo and RangeTblEntries that will + * be relevant. + */ + adding_relids = bms_difference(new_clause_relids, rinfo->clause_relids); + i = -1; + while ((i = bms_next_member(adding_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(!bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_add_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(!bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_add_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(adding_relids); + +#ifdef USE_ASSERT_CHECKING + + /* + * Verify that all indexes are set for common Relids. + */ + common_relids = bms_intersect(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(common_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + if (rinfo->eq_derives_index != -1) + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + } + bms_free(common_relids); +#endif + + /* + * Since we have done everything to adjust the indexes, we can set the + * clause_relids field. + */ + rinfo->clause_relids = new_clause_relids; +} + /* * get_eclass_for_sort_expr @@ -730,8 +903,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = collation; newec->ec_members = NIL; - newec->ec_sources = NIL; - newec->ec_derives = NIL; + newec->ec_source_indexes = NULL; + newec->ec_derive_indexes = NULL; newec->ec_relids = NULL; newec->ec_has_const = false; newec->ec_has_volatile = contain_volatile_functions((Node *) expr); @@ -1109,7 +1282,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources or ec_derives for matches. It doesn't really + * don't search eq_sources or eq_derives for matches. It doesn't really * seem worth the trouble to do so. */ void @@ -1198,6 +1371,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *const_em = NULL; ListCell *lc; + int i; /* * In the trivial case where we just had one "var = const" clause, push @@ -1207,9 +1381,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, * equivalent to the old one. */ if (list_length(ec->ec_members) == 2 && - list_length(ec->ec_sources) == 1) + bms_get_singleton_member(ec->ec_source_indexes, &i)) { - RestrictInfo *restrictinfo = (RestrictInfo *) linitial(ec->ec_sources); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, root->eq_sources, i); distribute_restrictinfo_to_rels(root, restrictinfo); return; @@ -1267,9 +1441,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the correct - * markings for a mergejoinable clause, and save it in ec_derives. (We + * markings for a mergejoinable clause, and save it in eq_derives. (We * will not re-use such clauses directly, but selectivity estimation - * may consult the list later. Note that this use of ec_derives does + * may consult the list later. Note that this use of eq_derives does * not overlap with its use for join clauses, since we never generate * join clauses from an ec_has_const eclass.) */ @@ -1279,7 +1453,8 @@ generate_base_implied_equalities_const(PlannerInfo *root, rinfo->left_ec = rinfo->right_ec = ec; rinfo->left_em = cur_em; rinfo->right_em = const_em; - ec->ec_derives = lappend(ec->ec_derives, rinfo); + + add_eq_derive(root, ec, rinfo); } } } @@ -1345,7 +1520,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the * correct markings for a mergejoinable clause. We don't put it - * in ec_derives however; we don't currently need to re-find such + * in eq_derives however; we don't currently need to re-find such * clauses, and we don't want to clutter that list with non-join * clauses. */ @@ -1401,11 +1576,12 @@ static void generate_base_implied_equalities_broken(PlannerInfo *root, EquivalenceClass *ec) { - ListCell *lc; + int i = -1; - foreach(lc, ec->ec_sources) + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); if (ec->ec_has_const || bms_membership(restrictinfo->required_relids) != BMS_MULTIPLE) @@ -1446,11 +1622,11 @@ generate_base_implied_equalities_broken(PlannerInfo *root, * Because the same join clauses are likely to be needed multiple times as * we consider different join paths, we avoid generating multiple copies: * whenever we select a particular pair of EquivalenceMembers to join, - * we check to see if the pair matches any original clause (in ec_sources) - * or previously-built clause (in ec_derives). This saves memory and allows - * re-use of information cached in RestrictInfos. We also avoid generating - * commutative duplicates, i.e. if the algorithm selects "a.x = b.y" but - * we already have "b.y = a.x", we return the existing clause. + * we check to see if the pair matches any original clause in root's + * eq_sources or previously-built clause (in root's eq_derives). This saves + * memory and allows re-use of information cached in RestrictInfos. We also + * avoid generating commutative duplicates, i.e. if the algorithm selects + * "a.x = b.y" but we already have "b.y = a.x", we return the existing clause. * * If we are considering an outer join, sjinfo is the associated OJ info, * otherwise it can be NULL. @@ -1828,12 +2004,16 @@ generate_join_implied_equalities_broken(PlannerInfo *root, Relids nominal_inner_relids, RelOptInfo *inner_rel) { + Bitmapset *matching_es; List *result = NIL; - ListCell *lc; + int i; - foreach(lc, ec->ec_sources) + matching_es = get_ec_source_indexes(root, ec, nominal_join_relids); + i = -1; + while ((i = bms_next_member(matching_es, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); Relids clause_relids = restrictinfo->required_relids; if (bms_is_subset(clause_relids, nominal_join_relids) && @@ -1845,12 +2025,12 @@ generate_join_implied_equalities_broken(PlannerInfo *root, /* * If we have to translate, just brute-force apply adjust_appendrel_attrs * to all the RestrictInfos at once. This will result in returning - * RestrictInfos that are not listed in ec_derives, but there shouldn't be + * RestrictInfos that are not listed in eq_derives, but there shouldn't be * any duplication, and it's a sufficiently narrow corner case that we * shouldn't sweat too much over it anyway. * * Since inner_rel might be an indirect descendant of the baserel - * mentioned in the ec_sources clauses, we have to be prepared to apply + * mentioned in the eq_sources clauses, we have to be prepared to apply * multiple levels of Var translation. */ if (IS_OTHER_REL(inner_rel) && result != NIL) @@ -1912,10 +2092,11 @@ create_join_clause(PlannerInfo *root, EquivalenceMember *rightem, EquivalenceClass *parent_ec) { + Bitmapset *matches; RestrictInfo *rinfo; RestrictInfo *parent_rinfo = NULL; - ListCell *lc; MemoryContext oldcontext; + int i; /* * Search to see if we already built a RestrictInfo for this pair of @@ -1926,9 +2107,12 @@ create_join_clause(PlannerInfo *root, * it's not identical, it'd better have the same effects, or the operator * families we're using are broken. */ - foreach(lc, ec->ec_sources) + matches = bms_int_members(get_ec_source_indexes_strict(root, ec, leftem->em_relids), + get_ec_source_indexes_strict(root, ec, rightem->em_relids)); + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -1939,9 +2123,13 @@ create_join_clause(PlannerInfo *root, return rinfo; } - foreach(lc, ec->ec_derives) + matches = bms_int_members(get_ec_derive_indexes_strict(root, ec, leftem->em_relids), + get_ec_derive_indexes_strict(root, ec, rightem->em_relids)); + + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -2014,7 +2202,7 @@ create_join_clause(PlannerInfo *root, rinfo->left_em = leftem; rinfo->right_em = rightem; /* and save it for possible re-use */ - ec->ec_derives = lappend(ec->ec_derives, rinfo); + add_eq_derive(root, ec, rinfo); MemoryContextSwitchTo(oldcontext); @@ -2702,16 +2890,19 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, * Returns NULL if no such clause can be found. */ RestrictInfo * -find_derived_clause_for_ec_member(EquivalenceClass *ec, +find_derived_clause_for_ec_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em) { - ListCell *lc; + int i; Assert(ec->ec_has_const); Assert(!em->em_is_const); - foreach(lc, ec->ec_derives) + + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); /* * generate_base_implied_equalities_const will have put non-const @@ -3425,7 +3616,7 @@ have_relevant_eclass_joinclause(PlannerInfo *root, * surely be true if both of them overlap ec_relids.) * * Note we don't test ec_broken; if we did, we'd need a separate code - * path to look through ec_sources. Checking the membership anyway is + * path to look through eq_sources. Checking the membership anyway is * OK as a possibly-overoptimistic heuristic. * * We don't test ec_has_const either, even though a const eclass won't @@ -3513,7 +3704,7 @@ eclass_useful_for_merging(PlannerInfo *root, /* * Note we don't test ec_broken; if we did, we'd need a separate code path - * to look through ec_sources. Checking the members anyway is OK as a + * to look through eq_sources. Checking the members anyway is OK as a * possibly-overoptimistic heuristic. */ @@ -3666,3 +3857,240 @@ get_common_eclass_indexes(PlannerInfo *root, Relids relids1, Relids relids2) /* Calculate and return the common EC indexes, recycling the left input. */ return bms_int_members(rel1ecs, rel2ecs); } + +/* + * get_ec_source_indexes + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_esis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_esis, ec->ec_source_indexes); +} + +/* + * get_ec_source_indexes_strict + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *esis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + esis = bms_intersect(ec->ec_source_indexes, + rte->eclass_source_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + esis = bms_int_members(esis, rte->eclass_source_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return esis; +} + +/* + * get_ec_derive_indexes + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + * + * XXX is this function even needed? + */ +Bitmapset * +get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_edis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_edis, ec->ec_derive_indexes); +} + +/* + * get_ec_derive_indexes_strict + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *edis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + edis = bms_intersect(ec->ec_derive_indexes, + rte->eclass_derive_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + edis = bms_int_members(edis, rte->eclass_derive_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return edis; +} + +#ifdef USE_ASSERT_CHECKING +/* + * verify_eclass_indexes + * Verify that there are no missing references between RestrictInfos and + * EquivalenceMember's indexes, namely eclass_source_indexes and + * eclass_derive_indexes. If you modify these indexes, you should check + * them with this function. + */ +void +verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) +{ + ListCell *lc; + + /* + * All RestrictInfos in root->eq_sources must have references to + * eclass_source_indexes. + */ + foreach(lc, root->eq_sources) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_sources, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + } + } + + /* + * All RestrictInfos in root->eq_derives must have references to + * eclass_derive_indexes. + */ + foreach(lc, root->eq_derives) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_derives, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + } + } +} +#endif diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index c3fd4a81f8..527a6986b7 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -35,9 +35,10 @@ static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); static void remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); -static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, +static void remove_rel_from_restrictinfo(PlannerInfo *root, + RestrictInfo *rinfo, int relid, int ojrelid); -static void remove_rel_from_eclass(EquivalenceClass *ec, +static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static bool rel_supports_distinctness(PlannerInfo *root, RelOptInfo *rel); @@ -502,7 +503,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * that any such PHV is safe (and updated its ph_eval_at), so we * can just drop those references. */ - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); /* * Cross-check that the clause itself does not reference the @@ -531,7 +532,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) if (bms_is_member(relid, ec->ec_relids) || bms_is_member(ojrelid, ec->ec_relids)) - remove_rel_from_eclass(ec, relid, ojrelid); + remove_rel_from_eclass(root, ec, relid, ojrelid); } /* @@ -559,19 +560,28 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * we have to also clean up the sub-clauses. */ static void -remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) +remove_rel_from_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo, int relid, + int ojrelid) { + Relids new_clause_relids; + /* * initsplan.c is fairly cavalier about allowing RestrictInfos to share * relid sets with other RestrictInfos, and SpecialJoinInfos too. Make * sure this RestrictInfo has its own relid sets before we modify them. - * (In present usage, clause_relids is probably not shared, but - * required_relids could be; let's not assume anything.) + * The 'new_clause_relids' passed to update_clause_relids() must be a + * different instance from the current rinfo->clause_relids, so we make a + * copy of it. + */ + new_clause_relids = bms_copy(rinfo->clause_relids); + new_clause_relids = bms_del_member(new_clause_relids, relid); + new_clause_relids = bms_del_member(new_clause_relids, ojrelid); + update_clause_relids(root, rinfo, new_clause_relids); + + /* + * In present usage, required_relids could be shared, so we make a copy of + * it. */ - rinfo->clause_relids = bms_copy(rinfo->clause_relids); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, relid); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, ojrelid); - /* Likewise for required_relids */ rinfo->required_relids = bms_copy(rinfo->required_relids); rinfo->required_relids = bms_del_member(rinfo->required_relids, relid); rinfo->required_relids = bms_del_member(rinfo->required_relids, ojrelid); @@ -596,14 +606,14 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) { RestrictInfo *rinfo2 = lfirst_node(RestrictInfo, lc2); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } else { RestrictInfo *rinfo2 = castNode(RestrictInfo, orarg); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } } @@ -619,9 +629,11 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) * level(s). */ static void -remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) +remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, + int ojrelid) { ListCell *lc; + int i; /* Fix up the EC's overall relids */ ec->ec_relids = bms_del_member(ec->ec_relids, relid); @@ -648,11 +660,12 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) } /* Fix up the source clauses, in case we can re-use them later */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); } /* @@ -660,7 +673,7 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) * drop them. (At this point, any such clauses would be base restriction * clauses, which we'd not need anymore anyway.) */ - ec->ec_derives = NIL; + ec->ec_derive_indexes = NULL; } /* diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index b5827d3980..dfaee24430 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -651,6 +651,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, PlannerInfo *parent_root, root->multiexpr_params = NIL; root->join_domains = NIL; root->eq_classes = NIL; + root->eq_sources = NIL; + root->eq_derives = NIL; root->ec_merging_done = false; root->last_rinfo_serial = 0; root->all_result_relids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 969e257f70..4ede159395 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1042,6 +1042,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, subroot->multiexpr_params = NIL; subroot->join_domains = NIL; subroot->eq_classes = NIL; + subroot->eq_sources = NIL; + subroot->eq_derives = NIL; subroot->ec_merging_done = false; subroot->last_rinfo_serial = 0; subroot->all_result_relids = NULL; diff --git a/src/backend/optimizer/util/appendinfo.c b/src/backend/optimizer/util/appendinfo.c index 4989722637..4489c2e1bf 100644 --- a/src/backend/optimizer/util/appendinfo.c +++ b/src/backend/optimizer/util/appendinfo.c @@ -495,6 +495,8 @@ adjust_appendrel_attrs_mutator(Node *node, newinfo->right_bucketsize = -1; newinfo->left_mcvfreq = -1; newinfo->right_mcvfreq = -1; + newinfo->eq_sources_index = -1; + newinfo->eq_derives_index = -1; return (Node *) newinfo; } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5b..b5d3984219 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,6 +492,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); + /* + * We do not want to inherit the EquivalenceMember indexes of the parent + * to its child + */ + childrte->eclass_source_indexes = NULL; + childrte->eclass_derive_indexes = NULL; + Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 0b406e9334..59ba503ecb 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -247,6 +247,9 @@ make_restrictinfo_internal(PlannerInfo *root, restrictinfo->left_hasheqoperator = InvalidOid; restrictinfo->right_hasheqoperator = InvalidOid; + restrictinfo->eq_sources_index = -1; + restrictinfo->eq_derives_index = -1; + return restrictinfo; } @@ -403,6 +406,8 @@ commute_restrictinfo(RestrictInfo *rinfo, Oid comm_op) result->right_mcvfreq = rinfo->left_mcvfreq; result->left_hasheqoperator = InvalidOid; result->right_hasheqoperator = InvalidOid; + result->eq_sources_index = -1; + result->eq_derives_index = -1; return result; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 124d853e49..c6e8740c18 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1238,6 +1238,12 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); + Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources + * list for RestrictInfos that mention + * this relation */ + Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives + * list for RestrictInfos that mention + * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index b625694e81..6643c862f4 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -321,6 +321,12 @@ struct PlannerInfo /* list of active EquivalenceClasses */ List *eq_classes; + /* list of source RestrictInfos used to build EquivalenceClasses */ + List *eq_sources; + + /* list of RestrictInfos derived from EquivalenceClasses */ + List *eq_derives; + /* set true once ECs are canonical */ bool ec_merging_done; @@ -1390,6 +1396,24 @@ typedef struct JoinDomain * its comment for usage. The approach to lookup child members quickly is * described as iterate_child_rel_equivalences() comment. * + * At various locations in the query planner, we must search for source and + * derived RestrictInfos regarding a given EquivalenceClass. For the common + * case, an EquivalenceClass does not have a large number of RestrictInfos, + * however, in cases such as planning queries to partitioned tables, the + * number of members can become large. To maintain planning performance, we + * make use of a bitmap index to allow us to quickly find RestrictInfos in a + * given EquivalenceClass belonging to a given relation or set of relations. + * This is done by storing a list of RestrictInfos belonging to all + * EquivalenceClasses in PlannerInfo and storing a Bitmapset for each + * RelOptInfo which has a bit set for each RestrictInfo in that list which + * relates to the given relation. We also store a Bitmapset to mark all of + * the indexes in the PlannerInfo's list of RestrictInfos in the + * EquivalenceClass. We can quickly find the interesting indexes into the + * PlannerInfo's list by performing a bit-wise AND on the RelOptInfo's + * Bitmapset and the EquivalenceClasses. RestrictInfos must be looked up in + * PlannerInfo by this technique using the ec_source_indexes and + * ec_derive_indexes Bitmapsets. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1408,8 +1432,10 @@ typedef struct EquivalenceClass List *ec_opfamilies; /* btree operator family OIDs */ Oid ec_collation; /* collation, if datatypes are collatable */ List *ec_members; /* list of EquivalenceMembers */ - List *ec_sources; /* list of generating RestrictInfos */ - List *ec_derives; /* list of derived RestrictInfos */ + Bitmapset *ec_source_indexes; /* indexes into PlannerInfo's eq_sources + * list of generating RestrictInfos */ + Bitmapset *ec_derive_indexes; /* indexes into PlannerInfo's eq_derives + * list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except * for child members (see below) */ bool ec_has_const; /* any pseudoconstants in ec_members? */ @@ -2650,7 +2676,12 @@ typedef struct RestrictInfo /* number of base rels in clause_relids */ int num_base_rels pg_node_attr(equal_ignore); - /* The relids (varnos+varnullingrels) actually referenced in the clause: */ + /* + * The relids (varnos+varnullingrels) actually referenced in the clause. + * + * NOTE: This field must be updated through update_clause_relids(), not by + * setting the field directly. + */ Relids clause_relids pg_node_attr(equal_ignore); /* The set of relids required to evaluate the clause: */ @@ -2768,6 +2799,10 @@ typedef struct RestrictInfo /* hash equality operators used for memoize nodes, else InvalidOid */ Oid left_hasheqoperator pg_node_attr(equal_ignore); Oid right_hasheqoperator pg_node_attr(equal_ignore); + + /* the index within root->eq_sources and root->eq_derives */ + int eq_sources_index pg_node_attr(equal_ignore); + int eq_derives_index pg_node_attr(equal_ignore); } RestrictInfo; /* diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 3894ef0644..34a0b0d98a 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -128,6 +128,8 @@ extern bool process_equivalence(PlannerInfo *root, extern Expr *canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation); extern void reconsider_outer_join_clauses(PlannerInfo *root); +extern void update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids); extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Expr *expr, List *opfamilies, @@ -164,7 +166,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); -extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, +extern RestrictInfo *find_derived_clause_for_ec_member(PlannerInfo *root, + EquivalenceClass *ec, EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, @@ -203,6 +206,22 @@ extern bool eclass_useful_for_merging(PlannerInfo *root, extern bool is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist); extern bool is_redundant_with_indexclauses(RestrictInfo *rinfo, List *indexclauses); +extern Bitmapset *get_ec_source_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_source_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +#ifdef USE_ASSERT_CHECKING +extern void verify_eclass_indexes(PlannerInfo *root, + EquivalenceClass *ec); +#endif /* * pathkeys.c -- 2.45.2.windows.1 [application/octet-stream] v25-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch (14.4K, ../../CAJ2pMkaWUaM53zoTFaLb1M8TX-2xmH5xk7dzvfjo2hSvy_zg3g@mail.gmail.com/4-v25-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch) download | inline diff: From 69362f8f5369fd9768dfb9028472963bdfca8994 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 19 Jan 2024 11:43:29 +0900 Subject: [PATCH v25 3/5] Move EquivalenceClass indexes to PlannerInfo In the previous commit, the indexes, namely ec_[source|derive]_indexes, were in RestrictInfo. This was a workaround because RelOptInfo was the best place but some RelOptInfos can be NULL and we cannot store indexes for them. This commit introduces a new struct, EquivalenceClassIndexes. This struct exists in PlannerInfo and holds our indexes. This change prevents a serialization problem with RestirctInfo in the previous commit. --- src/backend/nodes/outfuncs.c | 2 - src/backend/nodes/readfuncs.c | 2 - src/backend/optimizer/path/equivclass.c | 83 ++++++++++++------------- src/backend/optimizer/util/inherit.c | 7 --- src/backend/optimizer/util/relnode.c | 6 ++ src/include/nodes/parsenodes.h | 6 -- src/include/nodes/pathnodes.h | 26 ++++++++ src/tools/pgindent/typedefs.list | 1 + 8 files changed, 74 insertions(+), 59 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 1b3cae4fca..f20fb765d2 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -570,8 +570,6 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); - WRITE_BITMAPSET_FIELD(eclass_source_indexes); - WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index ccf06860fe..b47950764a 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -431,8 +431,6 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_BITMAPSET_FIELD(eclass_source_indexes); - READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index f7d04c8f46..c52f241a27 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -614,10 +614,10 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, - source_idx); + index->source_indexes = bms_add_member(index->source_indexes, + source_idx); } } @@ -638,10 +638,10 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, - derive_idx); + index->derive_indexes = bms_add_member(index->derive_indexes, + derive_idx); } } @@ -680,22 +680,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(removing_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_del_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_del_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_del_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_del_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -709,22 +709,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(adding_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(!bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_add_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_add_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(!bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_add_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_add_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -739,14 +739,14 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(common_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); + index->source_indexes)); if (rinfo->eq_derives_index != -1) Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); + index->derive_indexes)); } bms_free(common_relids); #endif @@ -3874,9 +3874,9 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + rel_esis = bms_add_members(rel_esis, index->source_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3912,7 +3912,7 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3921,12 +3921,12 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ esis = bms_intersect(ec->ec_source_indexes, - rte->eclass_source_indexes); + index->source_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - esis = bms_int_members(esis, rte->eclass_source_indexes); + index = &root->eclass_indexes_array[i]; + esis = bms_int_members(esis, index->source_indexes); } } @@ -3963,9 +3963,9 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + rel_edis = bms_add_members(rel_edis, index->derive_indexes); } #ifdef USE_ASSERT_CHECKING @@ -4001,7 +4001,7 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -4010,12 +4010,12 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, - rte->eclass_derive_indexes); + index->derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - edis = bms_int_members(edis, rte->eclass_derive_indexes); + index = &root->eclass_indexes_array[i]; + edis = bms_int_members(edis, index->derive_indexes); } } @@ -4038,9 +4038,8 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, /* * verify_eclass_indexes * Verify that there are no missing references between RestrictInfos and - * EquivalenceMember's indexes, namely eclass_source_indexes and - * eclass_derive_indexes. If you modify these indexes, you should check - * them with this function. + * EquivalenceMember's indexes, namely source_indexes and derive_indexes. + * If you modify these indexes, you should check them with this function. */ void verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) @@ -4049,7 +4048,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) /* * All RestrictInfos in root->eq_sources must have references to - * eclass_source_indexes. + * source_indexes. */ foreach(lc, root->eq_sources) { @@ -4066,13 +4065,13 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].source_indexes)); } } /* * All RestrictInfos in root->eq_derives must have references to - * eclass_derive_indexes. + * derive_indexes. */ foreach(lc, root->eq_derives) { @@ -4089,7 +4088,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].derive_indexes)); } } } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index b5d3984219..3c2198ea5b 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,13 +492,6 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); - /* - * We do not want to inherit the EquivalenceMember indexes of the parent - * to its child - */ - childrte->eclass_source_indexes = NULL; - childrte->eclass_derive_indexes = NULL; - Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index f32575b949..c219de44d7 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -119,6 +119,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->simple_rte_array[rti++] = rte; } + root->eclass_indexes_array = (EquivalenceClassIndexes *) + palloc0(size * sizeof(EquivalenceClassIndexes)); + /* append_rel_array is not needed if there are no AppendRelInfos */ if (root->append_rel_list == NIL) { @@ -218,6 +221,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->top_parent_relid_array = NULL; } + root->eclass_indexes_array = + repalloc0_array(root->eclass_indexes_array, EquivalenceClassIndexes, root->simple_rel_array_size, new_size); + root->simple_rel_array_size = new_size; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index c6e8740c18..124d853e49 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1238,12 +1238,6 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); - Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources - * list for RestrictInfos that mention - * this relation */ - Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives - * list for RestrictInfos that mention - * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 6643c862f4..ed075b9184 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -192,6 +192,8 @@ typedef struct PlannerInfo PlannerInfo; #define HAVE_PLANNERINFO_TYPEDEF 1 #endif +struct EquivalenceClassIndexes; + struct PlannerInfo { pg_node_attr(no_copy_equal, no_read, no_query_jumble) @@ -248,6 +250,13 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * eclass_indexes_array is the same length as simple_rel_array and holds + * the indexes of the corresponding rels for faster lookups of + * RestrictInfo. See the EquivalenceClass comment for more details. + */ + struct EquivalenceClassIndexes *eclass_indexes_array pg_node_attr(read_write_ignore); + /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within @@ -1528,6 +1537,23 @@ typedef struct List *ec_members; /* parent and child members */ } EquivalenceChildMemberIterator; +/* + * EquivalenceClassIndexes + * + * As mentioned in the EquivalenceClass comment, we introduce a + * bitmapset-based indexing mechanism for faster lookups of RestrictInfo. This + * struct exists for each relation and holds the corresponding indexes. + */ +typedef struct EquivalenceClassIndexes +{ + Bitmapset *source_indexes; /* Indexes in PlannerInfo's eq_sources list + * for RestrictInfos that mention this + * relation */ + Bitmapset *derive_indexes; /* Indexes in PlannerInfo's eq_derives list + * for RestrictInfos that mention this + * relation */ +} EquivalenceClassIndexes; + /* * PathKeys * diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bdd6b7fdd9..eae14707d3 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -684,6 +684,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceChildMemberIterator EquivalenceClass +EquivalenceClassIndexes EquivalenceMember ErrorContextCallback ErrorData -- 2.45.2.windows.1 [application/octet-stream] v25-0004-Rename-add_eq_member-to-add_parent_eq_member.patch (5.2K, ../../CAJ2pMkaWUaM53zoTFaLb1M8TX-2xmH5xk7dzvfjo2hSvy_zg3g@mail.gmail.com/5-v25-0004-Rename-add_eq_member-to-add_parent_eq_member.patch) download | inline diff: From 5d8cfe71ef20302255f6dce14f9fc558567334c1 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Wed, 21 Aug 2024 13:54:59 +0900 Subject: [PATCH v25 4/5] Rename add_eq_member() to add_parent_eq_member() After our changes, add_eq_member() no longer creates and adds child EquivalenceMembers. This commit renames it to add_parent_eq_member() to clarify that it only creates parent members, and that we need to use make_eq_member() to handle child EquivalenceMembers. --- src/backend/optimizer/path/equivclass.c | 54 ++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index c52f241a27..39b18e7859 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -38,10 +38,10 @@ static EquivalenceMember *make_eq_member(EquivalenceClass *ec, JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype); -static EquivalenceMember *add_eq_member(EquivalenceClass *ec, - Expr *expr, Relids relids, - JoinDomain *jdomain, - Oid datatype); +static EquivalenceMember *add_parent_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + Oid datatype); static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo); static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, @@ -390,8 +390,8 @@ process_equivalence(PlannerInfo *root, else if (ec1) { /* Case 3: add item2 to ec1 */ - em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, item2_type); + em2 = add_parent_eq_member(ec1, item2, item2_relids, + jdomain, item2_type); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -408,8 +408,8 @@ process_equivalence(PlannerInfo *root, else if (ec2) { /* Case 3: add item1 to ec2 */ - em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, item1_type); + em1 = add_parent_eq_member(ec2, item1, item1_relids, + jdomain, item1_type); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -441,10 +441,10 @@ process_equivalence(PlannerInfo *root, ec->ec_min_security = restrictinfo->security_level; ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; - em1 = add_eq_member(ec, item1, item1_relids, - jdomain, item1_type); - em2 = add_eq_member(ec, item2, item2_relids, - jdomain, item2_type); + em1 = add_parent_eq_member(ec, item1, item1_relids, + jdomain, item1_type); + em2 = add_parent_eq_member(ec, item2, item2_relids, + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -579,7 +579,7 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, } /* - * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * add_parent_eq_member - build a new parent EquivalenceMember and add it to an EC * * Note: We don't have a function to add a child member like * add_child_eq_member() because how to do it depends on the relations they are @@ -587,8 +587,8 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, * add_child_join_rel_equivalences() to see how to add child members. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, Oid datatype) +add_parent_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) { EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, NULL, datatype); @@ -922,14 +922,14 @@ get_eclass_for_sort_expr(PlannerInfo *root, */ expr_relids = pull_varnos(root, (Node *) expr); - newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, opcintype); + newem = add_parent_eq_member(newec, copyObject(expr), expr_relids, + jdomain, opcintype); /* - * add_eq_member doesn't check for volatile functions, set-returning - * functions, aggregates, or window functions, but such could appear in - * sort expressions; so we have to check whether its const-marking was - * correct. + * add_parent_eq_member doesn't check for volatile functions, + * set-returning functions, aggregates, or window functions, but such + * could appear in sort expressions; so we have to check whether its + * const-marking was correct. */ if (newec->ec_has_const) { @@ -3225,12 +3225,12 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + add_parent_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); lc2 = lnext(setop_pathkeys, lc2); } -- 2.45.2.windows.1 [application/octet-stream] v25-0005-Resolve-conflict-with-commit-66c0185.patch (6.4K, ../../CAJ2pMkaWUaM53zoTFaLb1M8TX-2xmH5xk7dzvfjo2hSvy_zg3g@mail.gmail.com/6-v25-0005-Resolve-conflict-with-commit-66c0185.patch) download | inline diff: From 99d2bc0c39892db2c51ecc98f2d9caac449f072c Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Tue, 27 Aug 2024 13:20:29 +0900 Subject: [PATCH v25 5/5] Resolve conflict with commit 66c0185 This commit resolves a conflict with 66c0185, which added add_setop_child_rel_equivalences(). The function creates child EquivalenceMembers to efficiently implement UNION queries. This commit adjusts our optimization to handle this type of child EquivalenceMembers based on UNION parent-child relationships. --- src/backend/optimizer/path/equivclass.c | 51 ++++++++++++++++++++++--- src/backend/optimizer/util/inherit.c | 8 +++- src/backend/optimizer/util/relnode.c | 10 +++-- src/include/nodes/pathnodes.h | 2 +- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 39b18e7859..bd920fefe8 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -3208,7 +3208,9 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, { TargetEntry *tle = lfirst_node(TargetEntry, lc); EquivalenceMember *parent_em; + EquivalenceMember *child_em; PathKey *pk; + Index parent_relid; if (tle->resjunk) continue; @@ -3225,12 +3227,49 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_parent_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + child_em = make_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); + child_rel->eclass_child_members = + lappend(child_rel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated using + * 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + parent_em->em_child_relids = + bms_add_member(parent_em->em_child_relids, child_rel->relid); + + /* + * Make an UNION parent-child relationship between parent_em and + * child_rel->relid. We record this relationship in + * root->top_parent_relid_array, which generally has AppendRelInfo + * relationships. We use the same array here to retrieve UNION child + * members. + * + * XXX Here we treat the first member of parent_em->em_relids as a + * parent of child_rel. Is this correct? What happens if + * parent_em->em_relids has two or more members? + */ + parent_relid = bms_next_member(parent_em->em_relids, -1); + if (root->top_parent_relid_array == NULL) + { + /* + * If the array is NULL, allocate it here. + */ + root->top_parent_relid_array = (Index *) + palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[child_rel->relid] == -1 || + root->top_parent_relid_array[child_rel->relid] == parent_relid); + root->top_parent_relid_array[child_rel->relid] = parent_relid; lc2 = lnext(setop_pathkeys, lc2); } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5b..7b2390687e 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -582,9 +582,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, * Find a top parent rel's index and save it to top_parent_relid_array. */ if (root->top_parent_relid_array == NULL) + { root->top_parent_relid_array = - (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); - Assert(root->top_parent_relid_array[childRTindex] == 0); + (Index *) palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[childRTindex] == -1); topParentRTindex = parentRTindex; while (root->append_rel_array[topParentRTindex] != NULL && root->append_rel_array[topParentRTindex]->parent_relid != 0) diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index c219de44d7..5ea9a41008 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -133,7 +133,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); root->top_parent_relid_array = (Index *) - palloc0(size * sizeof(Index)); + palloc(size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * size); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -206,7 +208,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); root->top_parent_relid_array = - repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + repalloc_array(root->top_parent_relid_array, Index, new_size); + MemSet(root->top_parent_relid_array + root->simple_rel_array_size, -1, + sizeof(Index) * add_size); } else { @@ -1608,7 +1612,7 @@ find_relids_top_parents_slow(PlannerInfo *root, Relids relids) { int top_parent_relid = (int) top_parent_relid_array[i]; - if (top_parent_relid == 0) + if (top_parent_relid == -1) top_parent_relid = i; /* 'i' has no parents, so add itself */ else if (top_parent_relid != i) is_top_parent = false; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index ed075b9184..bbeef6a65a 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -260,7 +260,7 @@ struct PlannerInfo /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within - * simple_rel_array. The element can be zero if the rel has no parents, + * simple_rel_array. The element can be -1 if the rel has no parents, * i.e., is itself in a top-level. */ Index *top_parent_relid_array pg_node_attr(read_write_ignore); -- 2.45.2.windows.1 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-10-01 02:35 Yuya Watari <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-10-01 02:35 UTC (permalink / raw) To: PostgreSQL Developers <[email protected]>; +Cc: jian he <[email protected]>; Ashutosh Bapat <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> Hello, On Thu, Aug 29, 2024 at 2:34 PM Yuya Watari <[email protected]> wrote: > > The previous patches no longer apply to the master, so I rebased them. > I'm sorry for the delay. I noticed the patches do not apply to the current master. I have attached the rebased version. There are no changes besides the rebase. -- Best regards, Yuya Watari Attachments: [application/octet-stream] v26-0001-Speed-up-searches-for-child-EquivalenceMembers.patch (58.8K, ../../CAJ2pMkZSqTUMnBMks8B+z4OTQ=V6ohOE=PR7FbNCVJkH3rnsvA@mail.gmail.com/2-v26-0001-Speed-up-searches-for-child-EquivalenceMembers.patch) download | inline diff: From a927dd85a6253b51cbf228dcdc29078c715a6918 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:43:36 +0900 Subject: [PATCH v26 1/5] Speed up searches for child EquivalenceMembers Traditionally, child EquivalenceMembers were in the EquivalenceClass->ec_members. When we wanted to find some child members matching a request, we had to perform a linear search. This search became heavy when tables had many partitions, leading to much planning time. After this commit, child EquivalenceMembers no longer exist in ec_members. Instead, RelOptInfos have them. This change demonstrates a significant performance improvement in planning time. --- contrib/postgres_fdw/postgres_fdw.c | 22 +- src/backend/optimizer/path/equivclass.c | 414 ++++++++++++++++++++---- src/backend/optimizer/path/indxpath.c | 35 +- src/backend/optimizer/path/pathkeys.c | 9 +- src/backend/optimizer/plan/createplan.c | 57 ++-- src/backend/optimizer/util/inherit.c | 14 + src/backend/optimizer/util/relnode.c | 89 +++++ src/include/nodes/pathnodes.h | 61 ++++ src/include/optimizer/pathnode.h | 18 ++ src/include/optimizer/paths.h | 12 +- src/tools/pgindent/typedefs.list | 1 + 11 files changed, 619 insertions(+), 113 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index adc62576d1..93cbc53f03 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7831,13 +7831,21 @@ conversion_error_callback(void *arg) EquivalenceMember * find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) { - ListCell *lc; - PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + Relids top_parent_rel_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, rel->relids); /* * Note we require !bms_is_empty, else we'd accept constant @@ -7849,6 +7857,7 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) is_foreign_expr(root, rel, em->em_expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -7902,9 +7911,8 @@ find_em_for_rel_target(PlannerInfo *root, EquivalenceClass *ec, if (em->em_is_const) continue; - /* Ignore child members */ - if (em->em_is_child) - continue; + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* Match if same expression (after stripping relabel) */ em_expr = em->em_expr; diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 8f6f005ecb..b2ae2c1f7a 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -33,10 +33,14 @@ #include "utils/lsyscache.h" +static EquivalenceMember *make_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + EquivalenceMember *parent, + Oid datatype); static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, - EquivalenceMember *parent, Oid datatype); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, @@ -69,6 +73,11 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo); static JoinDomain *find_join_domain(PlannerInfo *root, Relids relids); +static void add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel); static Bitmapset *get_eclass_indexes_for_relids(PlannerInfo *root, Relids relids); static Bitmapset *get_common_eclass_indexes(PlannerInfo *root, Relids relids1, @@ -374,7 +383,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); @@ -391,7 +400,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); @@ -423,9 +432,9 @@ process_equivalence(PlannerInfo *root, ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; em1 = add_eq_member(ec, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); em2 = add_eq_member(ec, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -511,11 +520,16 @@ canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation) } /* - * add_eq_member - build a new EquivalenceMember and add it to an EC + * make_eq_member + * + * Build a new EquivalenceMember without adding it to an EC. If 'parent' + * parameter is NULL, the result will be a parent member, otherwise a child + * member. Note that child EquivalenceMembers should not be added to its + * parent EquivalenceClass. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) +make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) { EquivalenceMember *em = makeNode(EquivalenceMember); @@ -526,6 +540,8 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, em->em_datatype = datatype; em->em_jdomain = jdomain; em->em_parent = parent; + em->em_child_relids = NULL; + em->em_child_joinrel_relids = NULL; if (bms_is_empty(relids)) { @@ -546,11 +562,29 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, { ec->ec_relids = bms_add_members(ec->ec_relids, relids); } - ec->ec_members = lappend(ec->ec_members, em); return em; } +/* + * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * + * Note: We don't have a function to add a child member like + * add_child_eq_member() because how to do it depends on the relations they are + * translated from. See add_child_rel_equivalences() and + * add_child_join_rel_equivalences() to see how to add child members. + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + + ec->ec_members = lappend(ec->ec_members, em); + return em; +} + /* * get_eclass_for_sort_expr @@ -599,6 +633,17 @@ get_eclass_for_sort_expr(PlannerInfo *root, EquivalenceMember *newem; ListCell *lc1; MemoryContext oldcontext; + Relids top_parent_rel; + + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_rel = find_relids_top_parents(root, rel); /* * Ensure the expression exposes the correct type and collation. @@ -617,7 +662,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; /* * Never match to a volatile EC, except when we are looking at another @@ -632,16 +678,28 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (!equal(opfamilies, cur_ec->ec_opfamilies)) continue; - foreach(lc2, cur_ec->ec_members) + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel); /* - * Ignore child members unless they match the request. + * Ignore child members unless they match the request. If this + * EquivalenceMember is a child, i.e., translated above, it should + * match the request. We cannot assert this if a request is + * bms_is_subset(). */ - if (cur_em->em_is_child && - !bms_equal(cur_em->em_relids, rel)) - continue; + Assert(!cur_em->em_is_child || bms_equal(cur_em->em_relids, rel)); /* * Match constants only within the same JoinDomain (see @@ -654,6 +712,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, equal(expr, cur_em->em_expr)) return cur_ec; /* Match! */ } + dispose_eclass_child_member_iterator(&it); } /* No match; does caller want a NULL result? */ @@ -691,7 +750,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, expr_relids = pull_varnos(root, (Node *) expr); newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, NULL, opcintype); + jdomain, opcintype); /* * add_eq_member doesn't check for volatile functions, set-returning @@ -761,19 +820,25 @@ get_eclass_for_sort_expr(PlannerInfo *root, * Child EC members are ignored unless they belong to given 'relids'. */ EquivalenceMember * -find_ec_member_matching_expr(EquivalenceClass *ec, +find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, Expr *expr, Relids relids) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); /* We ignore binary-compatible relabeling on both ends */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); Expr *emexpr; /* @@ -783,6 +848,11 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -800,6 +870,7 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (equal(emexpr, expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -832,11 +903,17 @@ find_computable_ec_member(PlannerInfo *root, Relids relids, bool require_parallel_safe) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); List *exprvars; ListCell *lc2; @@ -847,6 +924,11 @@ find_computable_ec_member(PlannerInfo *root, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -880,6 +962,7 @@ find_computable_ec_member(PlannerInfo *root, return em; /* found usable expression */ } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -943,7 +1026,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, { Expr *targetexpr = (Expr *) lfirst(lc); - em = find_ec_member_matching_expr(ec, targetexpr, rel->relids); + em = find_ec_member_matching_expr(root, ec, targetexpr, rel->relids); if (!em) continue; @@ -1568,7 +1651,12 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *new_members = NIL; List *outer_members = NIL; List *inner_members = NIL; - ListCell *lc1; + Relids top_parent_join_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_join_relids = find_relids_top_parents(root, join_relids); /* * First, scan the EC to identify member values that are computable at the @@ -1579,9 +1667,14 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * as well as to at least one input member, plus enforce at least one * outer-rel member equal to at least one inner-rel member. */ - foreach(lc1, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + bms_is_subset(cur_em->em_relids, top_parent_join_relids)) + iterate_child_rel_equivalences(&it, root, ec, cur_em, join_relids); /* * We don't need to check explicitly for child EC members. This test @@ -1598,6 +1691,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, else new_members = lappend(new_members, cur_em); } + dispose_eclass_child_member_iterator(&it); /* * First, select the joinclause if needed. We can equate any one outer @@ -1615,6 +1709,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, Oid best_eq_op = InvalidOid; int best_score = -1; RestrictInfo *rinfo; + ListCell *lc1; foreach(lc1, outer_members) { @@ -1689,6 +1784,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *old_members = list_concat(outer_members, inner_members); EquivalenceMember *prev_em = NULL; RestrictInfo *rinfo; + ListCell *lc1; /* For now, arbitrarily take the first old_member as the one to use */ if (old_members) @@ -1696,7 +1792,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, foreach(lc1, new_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + cur_em = (EquivalenceMember *) lfirst(lc1); if (prev_em != NULL) { @@ -2409,6 +2505,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) if (matchleft && matchright) { cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); + Assert(!bms_is_empty(coal_em->em_relids)); return true; } @@ -2530,8 +2627,8 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, Oid opfamily) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); if (equal(item1, em->em_expr)) item1member = true; else if (equal(item2, em->em_expr)) @@ -2602,8 +2699,8 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); Var *var; - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* EM must be a Var, possibly with RelabelType */ var = (Var *) em->em_expr; @@ -2700,6 +2797,7 @@ add_child_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_rel->top_parent_relids; Relids child_relids = child_rel->relids; int i; + ListCell *lc; /* * EC merging should be complete already, so we can use the parent rel's @@ -2712,7 +2810,6 @@ add_child_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2725,15 +2822,9 @@ add_child_rel_equivalences(PlannerInfo *root, /* Sanity check eclass_indexes only contain ECs for parent_rel */ Assert(bms_is_subset(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2746,8 +2837,8 @@ add_child_rel_equivalences(PlannerInfo *root, * combinations of children. (But add_child_join_rel_equivalences * may add targeted combinations for partitionwise-join purposes.) */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * Consider only members that reference and can be computed at @@ -2763,6 +2854,7 @@ add_child_rel_equivalences(PlannerInfo *root, /* OK, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_rel->reloptkind == RELOPT_BASEREL) { @@ -2792,9 +2884,20 @@ add_child_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, + child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_relids = bms_add_member(cur_em->em_child_relids, + child_rel->relid); /* Record this EC index for the child rel */ child_rel->eclass_indexes = bms_add_member(child_rel->eclass_indexes, i); @@ -2823,6 +2926,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; MemoryContext oldcontext; + ListCell *lc; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2844,7 +2948,6 @@ add_child_join_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(matching_ecs, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2857,15 +2960,9 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Sanity check on get_eclass_indexes_for_relids result */ Assert(bms_overlap(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2874,8 +2971,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, * We consider only original EC members here, not * already-transformed child members. */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * We may ignore expressions that reference a single baserel, @@ -2890,6 +2987,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Yes, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_joinrel->reloptkind == RELOPT_JOINREL) { @@ -2920,9 +3018,21 @@ add_child_join_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_joinrel->eclass_child_members = + lappend(child_joinrel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_joinrel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_joinrel_relids = + bms_add_member(cur_em->em_child_joinrel_relids, + child_joinrel->join_rel_list_index); } } } @@ -2991,6 +3101,161 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, list_length(root->eq_classes) - 1); } +/* + * setup_eclass_child_member_iterator + * Setup an EquivalenceChildMemberIterator 'it' so that it can iterate over + * EquivalenceMembers in 'ec'. + * + * Once used, the caller should dispose of the iterator by calling + * dispose_eclass_child_member_iterator(). + */ +void +setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec) +{ + it->index = -1; + it->modified = false; + it->ec_members = ec->ec_members; +} + +/* + * eclass_child_member_iterator_next + * Get a next EquivalenceMember from an EquivalenceChildMemberIterator 'it' + * that was setup by setup_eclass_child_member_iterator(). NULL is returned + * if there are no members left. + */ +EquivalenceMember * +eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it) +{ + if (++it->index < list_length(it->ec_members)) + return list_nth_node(EquivalenceMember, it->ec_members, it->index); + return NULL; +} + +/* + * dispose_eclass_child_member_iterator + * Free any memory allocated by the iterator. + */ +void +dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it) +{ + /* + * XXX Should we use list_free()? I decided to use this style to take + * advantage of speculative execution. + */ + if (unlikely(it->modified)) + pfree(it->ec_members); +} + +/* + * add_transformed_child_version + * Add a transformed EquivalenceMember referencing the given child_rel to + * the iterator. + * + * This function is expected to be called only from + * iterate_child_rel_equivalences(). + */ +static void +add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel) +{ + ListCell *lc; + + foreach(lc, child_rel->eclass_child_members) + { + EquivalenceMember *child_em = lfirst_node(EquivalenceMember, lc); + + /* Skip unwanted EquivalenceMembers */ + if (child_em->em_parent != parent_em) + continue; + + /* + * If this is the first time the iterator's list has been modified, we + * need to make a copy of it. + */ + if (!it->modified) + { + it->ec_members = list_copy(it->ec_members); + it->modified = true; + } + + /* Add this child EquivalenceMember to the list */ + it->ec_members = lappend(it->ec_members, child_em); + } +} + +/* + * iterate_child_rel_equivalences + * Add transformed EquivalenceMembers referencing child rels in the given + * child_relids to the iterator. + * + * The transformation is done in add_transformed_child_version(). + */ +void +iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids) +{ + int i; + Relids matching_relids; + + /* The given EquivalenceMember should be parent */ + Assert(!parent_em->em_is_child); + + /* + * EquivalenceClass->ec_members has only parent EquivalenceMembers. Child + * members are translated using child RelOptInfos and stored in them. This + * is done in add_child_rel_equivalences() and + * add_child_join_rel_equivalences(). To retrieve child EquivalenceMembers + * of some parent, we need to know which RelOptInfos have such child + * members. This information is stored in indexes named em_child_relids + * and em_child_joinrel_relids. + * + * This function iterates over the child EquivalenceMembers that reference + * the given 'child_relids'. To do this, we first intersect 'child_relids' + * with these indexes. The result contains Relids of RelOptInfos that have + * child EquivalenceMembers we want to retrieve. Then we get the child + * members from the RelOptInfos using add_transformed_child_version(). + * + * We need to do these steps for each of the two indexes. + */ + + /* + * First, we translate simple rels. + */ + matching_relids = bms_intersect(parent_em->em_child_relids, + child_relids); + i = -1; + while ((i = bms_next_member(matching_relids, i)) >= 0) + { + /* Add transformed child version for this child rel */ + RelOptInfo *child_rel = root->simple_rel_array[i]; + + Assert(child_rel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_rel); + } + bms_free(matching_relids); + + /* + * Next, we try to translate join rels. + */ + i = -1; + while ((i = bms_next_member(parent_em->em_child_joinrel_relids, i)) >= 0) + { + /* Add transformed child version for this child join rel */ + RelOptInfo *child_joinrel = + list_nth_node(RelOptInfo, root->join_rel_list, i); + + Assert(child_joinrel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_joinrel); + } +} + /* * generate_implied_equalities_for_column @@ -3024,7 +3289,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, { List *result = NIL; bool is_child_rel = (rel->reloptkind == RELOPT_OTHER_MEMBER_REL); - Relids parent_relids; + Relids parent_relids, + top_parent_rel_relids; int i; /* Should be OK to rely on eclass_indexes */ @@ -3033,6 +3299,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, /* Indexes are available only on base or "other" member relations. */ Assert(IS_SIMPLE_REL(rel)); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + /* If it's a child rel, we'll need to know what its parent(s) are */ if (is_child_rel) parent_relids = find_childrel_parents(root, rel); @@ -3045,6 +3314,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); EquivalenceMember *cur_em; ListCell *lc2; + EquivalenceChildMemberIterator it; /* Sanity check eclass_indexes only contain ECs for rel */ Assert(is_child_rel || bms_is_subset(rel->relids, cur_ec->ec_relids)); @@ -3066,15 +3336,19 @@ generate_implied_equalities_for_column(PlannerInfo *root, * corner cases, so for now we live with just reporting the first * match. See also get_eclass_for_sort_expr.) */ - cur_em = NULL; - foreach(lc2, cur_ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - cur_em = (EquivalenceMember *) lfirst(lc2); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel->relids); if (bms_equal(cur_em->em_relids, rel->relids) && callback(root, rel, cur_ec, cur_em, callback_arg)) break; - cur_em = NULL; } + dispose_eclass_child_member_iterator(&it); if (!cur_em) continue; @@ -3089,8 +3363,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, Oid eq_op; RestrictInfo *rinfo; - if (other_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!other_em->em_is_child); /* Make sure it'll be a join to a different rel */ if (other_em == cur_em || @@ -3308,8 +3582,8 @@ eclass_useful_for_merging(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); if (!bms_overlap(cur_em->em_relids, relids)) return true; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index c0fcc7d78d..ce14ca324f 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -183,7 +183,7 @@ static IndexClause *expand_indexqual_rowcompare(PlannerInfo *root, IndexOptInfo *index, Oid expr_op, bool var_on_left); -static void match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +static void match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p); static Expr *match_clause_to_ordering_op(IndexOptInfo *index, @@ -927,7 +927,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * query_pathkeys will allow an incremental sort to be considered on * the index's partially sorted results. */ - match_pathkeys_to_index(index, root->query_pathkeys, + match_pathkeys_to_index(root, index, root->query_pathkeys, &orderbyclauses, &orderbyclausecols); if (list_length(root->query_pathkeys) == list_length(orderbyclauses)) @@ -3017,12 +3017,16 @@ expand_indexqual_rowcompare(PlannerInfo *root, * item in the given 'pathkeys' list. */ static void -match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p) { + Relids top_parent_index_relids; ListCell *lc1; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_index_relids = find_relids_top_parents(root, index->rel->relids); + *orderby_clauses_p = NIL; /* set default results */ *clause_columns_p = NIL; @@ -3034,7 +3038,8 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, { PathKey *pathkey = (PathKey *) lfirst(lc1); bool found = false; - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *member; /* Pathkey must request default sort order for the target opfamily */ @@ -3054,15 +3059,30 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, * be considered to match more than one pathkey list, which is OK * here. See also get_eclass_for_sort_expr.) */ - foreach(lc2, pathkey->pk_eclass->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, pathkey->pk_eclass); + while ((member = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *member = (EquivalenceMember *) lfirst(lc2); int indexcol; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + bms_equal(member->em_relids, top_parent_index_relids)) + iterate_child_rel_equivalences(&it, root, pathkey->pk_eclass, member, + index->rel->relids); + /* No possibility of match if it references other relations */ - if (!bms_equal(member->em_relids, index->rel->relids)) + if (!member->em_is_child && + !bms_equal(member->em_relids, index->rel->relids)) continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(bms_equal(member->em_relids, index->rel->relids)); + /* * We allow any column of the index to match each pathkey; they * don't have to match left-to-right as you might expect. This is @@ -3091,6 +3111,7 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, if (found) /* don't want to look at remaining members */ break; } + dispose_eclass_child_member_iterator(&it); /* * Return the matches found so far when this pathkey couldn't be diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 035bbaa385..d8a1f904d3 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -1150,8 +1150,8 @@ convert_subquery_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid sub_expr_coll = sub_eclass->ec_collation; ListCell *k; - if (sub_member->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!sub_member->em_is_child); foreach(k, subquery_tlist) { @@ -1707,8 +1707,11 @@ select_outer_pathkeys_for_merge(PlannerInfo *root, { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); + /* Potential future join partner? */ - if (!em->em_is_const && !em->em_is_child && + if (!em->em_is_const && !bms_overlap(em->em_relids, joinrel->relids)) score++; } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index bb45ef318f..11d3fe9ddd 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -262,7 +262,9 @@ static IncrementalSort *make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); -static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -271,9 +273,11 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); -static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Sort *make_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids); -static IncrementalSort *make_incrementalsort_from_pathkeys(Plan *lefttree, +static IncrementalSort *make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, int nPresortedCols); static Sort *make_sort_from_groupcols(List *groupcls, AttrNumber *grpColIdx, @@ -295,7 +299,7 @@ static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); static Unique *make_unique_from_sortclauses(Plan *lefttree, List *distinctList); -static Unique *make_unique_from_pathkeys(Plan *lefttree, +static Unique *make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, int numCols); static Gather *make_gather(List *qptlist, List *qpqual, int nworkers, int rescan_param, bool single_copy, Plan *subplan); @@ -1283,7 +1287,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * function result; it must be the same plan node. However, we then * need to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys((Plan *) plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, (Plan *) plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1327,7 +1331,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * don't need an explicit sort, to make sure they are returning * the same sort key columns the Append expects. */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, nodeSortColIdx, false, @@ -1468,7 +1472,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * function result; it must be the same plan node. However, we then need * to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys(plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1499,7 +1503,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, subplan = create_plan_recurse(root, subpath, CP_EXACT_TLIST); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, node->sortColIdx, false, @@ -1978,7 +1982,7 @@ create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) Assert(pathkeys != NIL); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, best_path->subpath->parent->relids, gm_plan->sortColIdx, false, @@ -2192,7 +2196,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) * relids. Thus, if this sort path is based on a child relation, we must * pass its relids. */ - plan = make_sort_from_pathkeys(subplan, best_path->path.pathkeys, + plan = make_sort_from_pathkeys(root, subplan, best_path->path.pathkeys, IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -2216,7 +2220,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, /* See comments in create_sort_plan() above */ subplan = create_plan_recurse(root, best_path->spath.subpath, flags | CP_SMALL_TLIST); - plan = make_incrementalsort_from_pathkeys(subplan, + plan = make_incrementalsort_from_pathkeys(root, subplan, best_path->spath.path.pathkeys, IS_OTHER_REL(best_path->spath.subpath->parent) ? best_path->spath.path.parent->relids : NULL, @@ -2285,7 +2289,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag subplan = create_plan_recurse(root, best_path->subpath, flags | CP_LABEL_TLIST); - plan = make_unique_from_pathkeys(subplan, + plan = make_unique_from_pathkeys(root, subplan, best_path->path.pathkeys, best_path->numkeys); @@ -4523,7 +4527,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->outersortkeys) { Relids outer_relids = outer_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(outer_plan, + Sort *sort = make_sort_from_pathkeys(root, outer_plan, best_path->outersortkeys, outer_relids); @@ -4537,7 +4541,7 @@ create_mergejoin_plan(PlannerInfo *root, if (best_path->innersortkeys) { Relids inner_relids = inner_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(inner_plan, + Sort *sort = make_sort_from_pathkeys(root, inner_plan, best_path->innersortkeys, inner_relids); @@ -6161,7 +6165,7 @@ make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, * or a Result stacked atop lefttree). */ static Plan * -prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -6228,7 +6232,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ @@ -6256,7 +6260,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr already in tlist */ @@ -6272,7 +6276,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, /* * No matching tlist item; look for a computable expression. */ - em = find_computable_ec_member(NULL, ec, tlist, relids, false); + em = find_computable_ec_member(root, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; @@ -6343,7 +6347,8 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ static Sort * -make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) +make_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + Relids relids) { int numsortkeys; AttrNumber *sortColIdx; @@ -6352,7 +6357,7 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6378,8 +6383,9 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) * 'nPresortedCols' is the number of presorted columns in input tuples */ static IncrementalSort * -make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, - Relids relids, int nPresortedCols) +make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, + List *pathkeys, Relids relids, + int nPresortedCols) { int numsortkeys; AttrNumber *sortColIdx; @@ -6388,7 +6394,7 @@ make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6747,7 +6753,8 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) * as above, but use pathkeys to identify the sort columns and semantics */ static Unique * -make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) +make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + int numCols) { Unique *node = makeNode(Unique); Plan *plan = &node->plan; @@ -6810,7 +6817,7 @@ make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) foreach(j, plan->targetlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, NULL); + em = find_ec_member_matching_expr(root, ec, tle->expr, NULL); if (em) { /* found expr already in tlist */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index c5b906a9d4..3c2198ea5b 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -470,6 +470,7 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, RelationGetRelid(parentrel); Oid childOID = RelationGetRelid(childrel); RangeTblEntry *childrte; + Index topParentRTindex; Index childRTindex; AppendRelInfo *appinfo; TupleDesc child_tupdesc; @@ -577,6 +578,19 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, Assert(root->append_rel_array[childRTindex] == NULL); root->append_rel_array[childRTindex] = appinfo; + /* + * Find a top parent rel's index and save it to top_parent_relid_array. + */ + if (root->top_parent_relid_array == NULL) + root->top_parent_relid_array = + (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); + Assert(root->top_parent_relid_array[childRTindex] == 0); + topParentRTindex = parentRTindex; + while (root->append_rel_array[topParentRTindex] != NULL && + root->append_rel_array[topParentRTindex]->parent_relid != 0) + topParentRTindex = root->append_rel_array[topParentRTindex]->parent_relid; + root->top_parent_relid_array[childRTindex] = topParentRTindex; + /* * Build a PlanRowMark if parent is marked FOR UPDATE/SHARE. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index d7266e4cdb..f32575b949 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -123,11 +123,14 @@ setup_simple_rel_arrays(PlannerInfo *root) if (root->append_rel_list == NIL) { root->append_rel_array = NULL; + root->top_parent_relid_array = NULL; return; } root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); + root->top_parent_relid_array = (Index *) + palloc0(size * sizeof(Index)); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -148,6 +151,27 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array[child_relid] = appinfo; } + + /* + * Find a top parent rel's relid for each AppendRelInfo. We cannot do this + * in the last foreach loop because there may be multi-level parent-child + * relations. + */ + for (int i = 0; i < size; i++) + { + int top_parent_relid; + + if (root->append_rel_array[i] == NULL) + continue; + + top_parent_relid = root->append_rel_array[i]->parent_relid; + + while (root->append_rel_array[top_parent_relid] != NULL && + root->append_rel_array[top_parent_relid]->parent_relid != 0) + top_parent_relid = root->append_rel_array[top_parent_relid]->parent_relid; + + root->top_parent_relid_array[i] = top_parent_relid; + } } /* @@ -175,12 +199,25 @@ expand_planner_arrays(PlannerInfo *root, int add_size) repalloc0_array(root->simple_rte_array, RangeTblEntry *, root->simple_rel_array_size, new_size); if (root->append_rel_array) + { root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); + root->top_parent_relid_array = + repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + } else + { root->append_rel_array = palloc0_array(AppendRelInfo *, new_size); + /* + * We do not allocate top_parent_relid_array here so that + * find_relids_top_parents() can early find all of the given Relids + * are top-level. + */ + root->top_parent_relid_array = NULL; + } + root->simple_rel_array_size = new_size; } @@ -234,6 +271,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; + rel->eclass_child_members = NIL; rel->serverid = InvalidOid; if (rte->rtekind == RTE_RELATION) { @@ -629,6 +667,12 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) /* GEQO requires us to append the new joinrel to the end of the list! */ root->join_rel_list = lappend(root->join_rel_list, joinrel); + /* + * Store the index of this joinrel to use in + * add_child_join_rel_equivalences(). + */ + joinrel->join_rel_list_index = list_length(root->join_rel_list) - 1; + /* store it into the auxiliary hashtable if there is one. */ if (root->join_rel_hash) { @@ -741,6 +785,7 @@ build_join_rel(PlannerInfo *root, joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -928,6 +973,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -1490,6 +1536,7 @@ fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids) upperrel->cheapest_total_path = NULL; upperrel->cheapest_unique_path = NULL; upperrel->cheapest_parameterized_paths = NIL; + upperrel->eclass_child_members = NIL; /* XXX Is this required? */ root->upper_rels[kind] = lappend(root->upper_rels[kind], upperrel); @@ -1530,6 +1577,48 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } +/* + * find_relids_top_parents_slow + * Slow path of find_relids_top_parents() macro. + * + * Do not call this directly; use the macro instead. See the macro comment for + * more details. + */ +Relids +find_relids_top_parents_slow(PlannerInfo *root, Relids relids) +{ + Index *top_parent_relid_array = root->top_parent_relid_array; + Relids result; + bool is_top_parent; + int i; + + /* This function should be called in the slow path */ + Assert(top_parent_relid_array != NULL); + + result = NULL; + is_top_parent = true; + i = -1; + while ((i = bms_next_member(relids, i)) >= 0) + { + int top_parent_relid = (int) top_parent_relid_array[i]; + + if (top_parent_relid == 0) + top_parent_relid = i; /* 'i' has no parents, so add itself */ + else if (top_parent_relid != i) + is_top_parent = false; + result = bms_add_member(result, top_parent_relid); + } + + if (is_top_parent) + { + bms_free(result); + return NULL; + } + + return result; +} + + /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 07e2415398..a1f044d8a6 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -248,6 +248,14 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * top_parent_relid_array is the same length as simple_rel_array and holds + * the top-level parent indexes of the corresponding rels within + * simple_rel_array. The element can be zero if the rel has no parents, + * i.e., is itself in a top-level. + */ + Index *top_parent_relid_array pg_node_attr(read_write_ignore); + /* * all_baserels is a Relids set of all base relids (but not joins or * "other" rels) in the query. This is computed in deconstruct_jointree. @@ -957,6 +965,17 @@ typedef struct RelOptInfo /* Bitmask of optional features supported by the table AM */ uint32 amflags; + /* + * information about a join rel + */ + /* index in root->join_rel_list of this rel */ + Index join_rel_list_index; + + /* + * EquivalenceMembers referencing this rel + */ + List *eclass_child_members; + /* * Information about foreign tables and foreign joins */ @@ -1371,6 +1390,12 @@ typedef struct JoinDomain * entry: consider SELECT random() AS a, random() AS b ... ORDER BY b,a. * So we record the SortGroupRef of the originating sort clause. * + * EquivalenceClass->ec_members can only have parent members, and child members + * are stored in RelOptInfos, from which those child members are translated. To + * lookup child EquivalenceMembers, we use EquivalenceChildMemberIterator. See + * its comment for usage. The approach to lookup child members quickly is + * described as iterate_child_rel_equivalences() comment. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1443,10 +1468,46 @@ typedef struct EquivalenceMember bool em_is_child; /* derived version for a child relation? */ Oid em_datatype; /* the "nominal type" used by the opfamily */ JoinDomain *em_jdomain; /* join domain containing the source clause */ + Relids em_child_relids; /* all relids of child rels */ + Bitmapset *em_child_joinrel_relids; /* indexes in root->join_rel_list + * of join rel children */ /* if em_is_child is true, this links to corresponding EM for top parent */ struct EquivalenceMember *em_parent pg_node_attr(read_write_ignore); } EquivalenceMember; +/* + * EquivalenceChildMemberIterator + * + * EquivalenceClass has only parent EquivalenceMembers, so we need to translate + * child members if necessary. EquivalenceChildMemberIterator provides a way to + * iterate over translated child members during the loop in addition to all of + * the parent EquivalenceMembers. + * + * The most common way to use this struct is as follows: + * ----- + * EquivalenceClass *ec = given; + * Relids rel = given; + * EquivalenceChildMemberIterator it; + * EquivalenceMember *em; + * + * setup_eclass_child_member_iterator(&it, ec); + * while ((em = eclass_child_member_iterator_next(&it)) != NULL) + * { + * use em ...; + * if (we need to iterate over child EquivalenceMembers) + * iterate_child_rel_equivalences(&it, root, ec, em, rel); + * use em ...; + * } + * dispose_eclass_child_member_iterator(&it); + * ----- + */ +typedef struct +{ + int index; /* current index within 'ec_members'. */ + bool modified; /* is 'ec_members' a newly allocated one? */ + List *ec_members; /* parent and child members */ +} EquivalenceChildMemberIterator; + /* * PathKeys * diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 1035e6560c..5e79cf1f63 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -332,6 +332,24 @@ extern Relids min_join_parameterization(PlannerInfo *root, extern RelOptInfo *fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids); extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); + +/* + * find_relids_top_parents + * Compute the set of top-parent relids of rel. + * + * Replaces all Relids appearing in the given 'relids' as their top-level + * parents. The result will be NULL if and only if all of the given relids are + * top-level. + * + * The motivation for having this feature as a macro rather than a function is + * that Relids are top-level in most cases. We can quickly determine when + * root->top_parent_relid_array is NULL. + */ +#define find_relids_top_parents(root, relids) \ + (likely((root)->top_parent_relid_array == NULL) \ + ? NULL : find_relids_top_parents_slow(root, relids)) +extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); + extern ParamPathInfo *get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, Relids required_outer); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 54869d4401..50812e3a5d 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -137,7 +137,8 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Index sortref, Relids rel, bool create_it); -extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec, +extern EquivalenceMember *find_ec_member_matching_expr(PlannerInfo *root, + EquivalenceClass *ec, Expr *expr, Relids relids); extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root, @@ -179,6 +180,15 @@ extern void add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, List *child_tlist, List *setop_pathkeys); +extern void setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec); +extern EquivalenceMember *eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it); +extern void dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it); +extern void iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids); extern List *generate_implied_equalities_for_column(PlannerInfo *root, RelOptInfo *rel, ec_matches_callback_type callback, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 5fabb127d7..623a770b0e 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -683,6 +683,7 @@ EphemeralNamedRelation EphemeralNamedRelationData EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData +EquivalenceChildMemberIterator EquivalenceClass EquivalenceMember ErrorContextCallback -- 2.45.2.windows.1 [application/octet-stream] v26-0002-Introduce-indexes-for-RestrictInfo.patch (42.9K, ../../CAJ2pMkZSqTUMnBMks8B+z4OTQ=V6ohOE=PR7FbNCVJkH3rnsvA@mail.gmail.com/3-v26-0002-Introduce-indexes-for-RestrictInfo.patch) download | inline diff: From 745b304ecfb4cffe1db1ecc7b64f6069b91f8adc Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:44:25 +0900 Subject: [PATCH v26 2/5] Introduce indexes for RestrictInfo This commit adds indexes to speed up searches for RestrictInfos. When there are many child partitions and we have to perform planning for join operations on the tables, we have to handle a large number of RestrictInfos, resulting in a long planning time. This commit adds ec_[source|derive]_indexes to speed up the search. We can use the indexes to filter out unwanted RestrictInfos, and improve the planning performance. Author: David Rowley <[email protected]> and me, and includes rebase by Alena Rybakina <[email protected]> [1]. [1] https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/nodes/outfuncs.c | 6 +- src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/path/equivclass.c | 516 ++++++++++++++++++++-- src/backend/optimizer/plan/analyzejoins.c | 49 +- src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/appendinfo.c | 2 + src/backend/optimizer/util/inherit.c | 7 + src/backend/optimizer/util/restrictinfo.c | 5 + src/include/nodes/parsenodes.h | 6 + src/include/nodes/pathnodes.h | 41 +- src/include/optimizer/paths.h | 21 +- 13 files changed, 593 insertions(+), 69 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 9827cf16be..bd6a79b57a 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -466,8 +466,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_opfamilies); WRITE_OID_FIELD(ec_collation); WRITE_NODE_FIELD(ec_members); - WRITE_NODE_FIELD(ec_sources); - WRITE_NODE_FIELD(ec_derives); + WRITE_BITMAPSET_FIELD(ec_source_indexes); + WRITE_BITMAPSET_FIELD(ec_derive_indexes); WRITE_BITMAPSET_FIELD(ec_relids); WRITE_BOOL_FIELD(ec_has_const); WRITE_BOOL_FIELD(ec_has_volatile); @@ -573,6 +573,8 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); + WRITE_BITMAPSET_FIELD(eclass_source_indexes); + WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index be5f19dd7f..70864fc1b2 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -434,6 +434,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); + READ_BITMAPSET_FIELD(eclass_source_indexes); + READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index e1523d15df..f8c04ea6b0 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5792,7 +5792,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, if (ec && ec->ec_has_const) { EquivalenceMember *em = fkinfo->fk_eclass_member[i]; - RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + RestrictInfo *rinfo = find_derived_clause_for_ec_member(root, + ec, em); if (rinfo) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index b2ae2c1f7a..7544ea93f2 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -42,6 +42,10 @@ static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, Oid datatype); +static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); +static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); static bool is_exprlist_member(Expr *node, List *exprs); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); @@ -320,7 +324,6 @@ process_equivalence(PlannerInfo *root, /* If case 1, nothing to do, except add to sources */ if (ec1 == ec2) { - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -331,6 +334,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); return true; } @@ -351,8 +356,10 @@ process_equivalence(PlannerInfo *root, * be found. */ ec1->ec_members = list_concat(ec1->ec_members, ec2->ec_members); - ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); - ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); + ec1->ec_source_indexes = bms_join(ec1->ec_source_indexes, + ec2->ec_source_indexes); + ec1->ec_derive_indexes = bms_join(ec1->ec_derive_indexes, + ec2->ec_derive_indexes); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); ec1->ec_has_const |= ec2->ec_has_const; /* can't need to set has_volatile */ @@ -364,10 +371,9 @@ process_equivalence(PlannerInfo *root, root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; - ec2->ec_sources = NIL; - ec2->ec_derives = NIL; + ec2->ec_source_indexes = NULL; + ec2->ec_derive_indexes = NULL; ec2->ec_relids = NULL; - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -378,13 +384,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec1) { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, jdomain, item2_type); - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -395,13 +402,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec2) { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, jdomain, item1_type); - ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -412,6 +420,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec2, restrictinfo); } else { @@ -421,8 +431,8 @@ process_equivalence(PlannerInfo *root, ec->ec_opfamilies = opfamilies; ec->ec_collation = collation; ec->ec_members = NIL; - ec->ec_sources = list_make1(restrictinfo); - ec->ec_derives = NIL; + ec->ec_source_indexes = NULL; + ec->ec_derive_indexes = NULL; ec->ec_relids = NULL; ec->ec_has_const = false; ec->ec_has_volatile = false; @@ -444,6 +454,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec, restrictinfo); } return true; @@ -585,6 +597,167 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, return em; } +/* + * add_eq_source - add 'rinfo' in eq_sources for this 'ec' + */ +static void +add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int source_idx = list_length(root->eq_sources); + int i; + + ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); + root->eq_sources = lappend(root->eq_sources, rinfo); + Assert(rinfo->eq_sources_index == -1); + rinfo->eq_sources_index = source_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + source_idx); + } +} + +/* + * add_eq_derive - add 'rinfo' in eq_derives for this 'ec' + */ +static void +add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int derive_idx = list_length(root->eq_derives); + int i; + + ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); + root->eq_derives = lappend(root->eq_derives, rinfo); + Assert(rinfo->eq_derives_index == -1); + rinfo->eq_derives_index = derive_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + derive_idx); + } +} + +/* + * update_clause_relids + * Update RestrictInfo->clause_relids with adjusting the relevant indexes. + * The clause_relids field must be updated through this function, not by + * setting the field directly. + */ +void +update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids) +{ + int i; + Relids removing_relids; + Relids adding_relids; +#ifdef USE_ASSERT_CHECKING + Relids common_relids; +#endif + + /* + * If there is nothing to do, we return immediately after setting the + * clause_relids field. + */ + if (rinfo->eq_sources_index == -1 && rinfo->eq_derives_index == -1) + { + rinfo->clause_relids = new_clause_relids; + return; + } + + /* + * Remove any references between this RestrictInfo and RangeTblEntries + * that are no longer relevant. + */ + removing_relids = bms_difference(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(removing_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(removing_relids); + + /* + * Add references between this RestrictInfo and RangeTblEntries that will + * be relevant. + */ + adding_relids = bms_difference(new_clause_relids, rinfo->clause_relids); + i = -1; + while ((i = bms_next_member(adding_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(!bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_add_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(!bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_add_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(adding_relids); + +#ifdef USE_ASSERT_CHECKING + + /* + * Verify that all indexes are set for common Relids. + */ + common_relids = bms_intersect(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(common_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + if (rinfo->eq_derives_index != -1) + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + } + bms_free(common_relids); +#endif + + /* + * Since we have done everything to adjust the indexes, we can set the + * clause_relids field. + */ + rinfo->clause_relids = new_clause_relids; +} + /* * get_eclass_for_sort_expr @@ -730,8 +903,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = collation; newec->ec_members = NIL; - newec->ec_sources = NIL; - newec->ec_derives = NIL; + newec->ec_source_indexes = NULL; + newec->ec_derive_indexes = NULL; newec->ec_relids = NULL; newec->ec_has_const = false; newec->ec_has_volatile = contain_volatile_functions((Node *) expr); @@ -1113,7 +1286,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources or ec_derives for matches. It doesn't really + * don't search eq_sources or eq_derives for matches. It doesn't really * seem worth the trouble to do so. */ void @@ -1206,6 +1379,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *const_em = NULL; ListCell *lc; + int i; /* * In the trivial case where we just had one "var = const" clause, push @@ -1215,9 +1389,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, * equivalent to the old one. */ if (list_length(ec->ec_members) == 2 && - list_length(ec->ec_sources) == 1) + bms_get_singleton_member(ec->ec_source_indexes, &i)) { - RestrictInfo *restrictinfo = (RestrictInfo *) linitial(ec->ec_sources); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, root->eq_sources, i); distribute_restrictinfo_to_rels(root, restrictinfo); return; @@ -1275,9 +1449,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the correct - * markings for a mergejoinable clause, and save it in ec_derives. (We + * markings for a mergejoinable clause, and save it in eq_derives. (We * will not re-use such clauses directly, but selectivity estimation - * may consult the list later. Note that this use of ec_derives does + * may consult the list later. Note that this use of eq_derives does * not overlap with its use for join clauses, since we never generate * join clauses from an ec_has_const eclass.) */ @@ -1287,7 +1461,8 @@ generate_base_implied_equalities_const(PlannerInfo *root, rinfo->left_ec = rinfo->right_ec = ec; rinfo->left_em = cur_em; rinfo->right_em = const_em; - ec->ec_derives = lappend(ec->ec_derives, rinfo); + + add_eq_derive(root, ec, rinfo); } } } @@ -1353,7 +1528,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the * correct markings for a mergejoinable clause. We don't put it - * in ec_derives however; we don't currently need to re-find such + * in eq_derives however; we don't currently need to re-find such * clauses, and we don't want to clutter that list with non-join * clauses. */ @@ -1410,11 +1585,12 @@ static void generate_base_implied_equalities_broken(PlannerInfo *root, EquivalenceClass *ec) { - ListCell *lc; + int i = -1; - foreach(lc, ec->ec_sources) + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); if (ec->ec_has_const || bms_membership(restrictinfo->required_relids) != BMS_MULTIPLE) @@ -1455,11 +1631,11 @@ generate_base_implied_equalities_broken(PlannerInfo *root, * Because the same join clauses are likely to be needed multiple times as * we consider different join paths, we avoid generating multiple copies: * whenever we select a particular pair of EquivalenceMembers to join, - * we check to see if the pair matches any original clause (in ec_sources) - * or previously-built clause (in ec_derives). This saves memory and allows - * re-use of information cached in RestrictInfos. We also avoid generating - * commutative duplicates, i.e. if the algorithm selects "a.x = b.y" but - * we already have "b.y = a.x", we return the existing clause. + * we check to see if the pair matches any original clause in root's + * eq_sources or previously-built clause (in root's eq_derives). This saves + * memory and allows re-use of information cached in RestrictInfos. We also + * avoid generating commutative duplicates, i.e. if the algorithm selects + * "a.x = b.y" but we already have "b.y = a.x", we return the existing clause. * * If we are considering an outer join, sjinfo is the associated OJ info, * otherwise it can be NULL. @@ -1837,12 +2013,16 @@ generate_join_implied_equalities_broken(PlannerInfo *root, Relids nominal_inner_relids, RelOptInfo *inner_rel) { + Bitmapset *matching_es; List *result = NIL; - ListCell *lc; + int i; - foreach(lc, ec->ec_sources) + matching_es = get_ec_source_indexes(root, ec, nominal_join_relids); + i = -1; + while ((i = bms_next_member(matching_es, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); Relids clause_relids = restrictinfo->required_relids; if (bms_is_subset(clause_relids, nominal_join_relids) && @@ -1854,12 +2034,12 @@ generate_join_implied_equalities_broken(PlannerInfo *root, /* * If we have to translate, just brute-force apply adjust_appendrel_attrs * to all the RestrictInfos at once. This will result in returning - * RestrictInfos that are not listed in ec_derives, but there shouldn't be + * RestrictInfos that are not listed in eq_derives, but there shouldn't be * any duplication, and it's a sufficiently narrow corner case that we * shouldn't sweat too much over it anyway. * * Since inner_rel might be an indirect descendant of the baserel - * mentioned in the ec_sources clauses, we have to be prepared to apply + * mentioned in the eq_sources clauses, we have to be prepared to apply * multiple levels of Var translation. */ if (IS_OTHER_REL(inner_rel) && result != NIL) @@ -1921,10 +2101,11 @@ create_join_clause(PlannerInfo *root, EquivalenceMember *rightem, EquivalenceClass *parent_ec) { + Bitmapset *matches; RestrictInfo *rinfo; RestrictInfo *parent_rinfo = NULL; - ListCell *lc; MemoryContext oldcontext; + int i; /* * Search to see if we already built a RestrictInfo for this pair of @@ -1935,9 +2116,12 @@ create_join_clause(PlannerInfo *root, * it's not identical, it'd better have the same effects, or the operator * families we're using are broken. */ - foreach(lc, ec->ec_sources) + matches = bms_int_members(get_ec_source_indexes_strict(root, ec, leftem->em_relids), + get_ec_source_indexes_strict(root, ec, rightem->em_relids)); + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -1948,9 +2132,13 @@ create_join_clause(PlannerInfo *root, return rinfo; } - foreach(lc, ec->ec_derives) + matches = bms_int_members(get_ec_derive_indexes_strict(root, ec, leftem->em_relids), + get_ec_derive_indexes_strict(root, ec, rightem->em_relids)); + + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -2023,7 +2211,7 @@ create_join_clause(PlannerInfo *root, rinfo->left_em = leftem; rinfo->right_em = rightem; /* and save it for possible re-use */ - ec->ec_derives = lappend(ec->ec_derives, rinfo); + add_eq_derive(root, ec, rinfo); MemoryContextSwitchTo(oldcontext); @@ -2749,16 +2937,19 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, * Returns NULL if no such clause can be found. */ RestrictInfo * -find_derived_clause_for_ec_member(EquivalenceClass *ec, +find_derived_clause_for_ec_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em) { - ListCell *lc; + int i; Assert(ec->ec_has_const); Assert(!em->em_is_const); - foreach(lc, ec->ec_derives) + + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); /* * generate_base_implied_equalities_const will have put non-const @@ -3472,7 +3663,7 @@ have_relevant_eclass_joinclause(PlannerInfo *root, * surely be true if both of them overlap ec_relids.) * * Note we don't test ec_broken; if we did, we'd need a separate code - * path to look through ec_sources. Checking the membership anyway is + * path to look through eq_sources. Checking the membership anyway is * OK as a possibly-overoptimistic heuristic. * * We don't test ec_has_const either, even though a const eclass won't @@ -3560,7 +3751,7 @@ eclass_useful_for_merging(PlannerInfo *root, /* * Note we don't test ec_broken; if we did, we'd need a separate code path - * to look through ec_sources. Checking the members anyway is OK as a + * to look through eq_sources. Checking the members anyway is OK as a * possibly-overoptimistic heuristic. */ @@ -3717,3 +3908,240 @@ get_common_eclass_indexes(PlannerInfo *root, Relids relids1, Relids relids2) /* Calculate and return the common EC indexes, recycling the left input. */ return bms_int_members(rel1ecs, rel2ecs); } + +/* + * get_ec_source_indexes + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_esis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_esis, ec->ec_source_indexes); +} + +/* + * get_ec_source_indexes_strict + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *esis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + esis = bms_intersect(ec->ec_source_indexes, + rte->eclass_source_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + esis = bms_int_members(esis, rte->eclass_source_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return esis; +} + +/* + * get_ec_derive_indexes + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + * + * XXX is this function even needed? + */ +Bitmapset * +get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_edis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_edis, ec->ec_derive_indexes); +} + +/* + * get_ec_derive_indexes_strict + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *edis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + edis = bms_intersect(ec->ec_derive_indexes, + rte->eclass_derive_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + edis = bms_int_members(edis, rte->eclass_derive_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return edis; +} + +#ifdef USE_ASSERT_CHECKING +/* + * verify_eclass_indexes + * Verify that there are no missing references between RestrictInfos and + * EquivalenceMember's indexes, namely eclass_source_indexes and + * eclass_derive_indexes. If you modify these indexes, you should check + * them with this function. + */ +void +verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) +{ + ListCell *lc; + + /* + * All RestrictInfos in root->eq_sources must have references to + * eclass_source_indexes. + */ + foreach(lc, root->eq_sources) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_sources, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + } + } + + /* + * All RestrictInfos in root->eq_derives must have references to + * eclass_derive_indexes. + */ + foreach(lc, root->eq_derives) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_derives, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + } + } +} +#endif diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 928d926645..c8f7435909 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -36,9 +36,10 @@ static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); static void remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); -static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, +static void remove_rel_from_restrictinfo(PlannerInfo *root, + RestrictInfo *rinfo, int relid, int ojrelid); -static void remove_rel_from_eclass(EquivalenceClass *ec, +static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static bool rel_supports_distinctness(PlannerInfo *root, RelOptInfo *rel); @@ -476,7 +477,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * that any such PHV is safe (and updated its ph_eval_at), so we * can just drop those references. */ - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); /* * Cross-check that the clause itself does not reference the @@ -505,7 +506,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) if (bms_is_member(relid, ec->ec_relids) || bms_is_member(ojrelid, ec->ec_relids)) - remove_rel_from_eclass(ec, relid, ojrelid); + remove_rel_from_eclass(root, ec, relid, ojrelid); } /* @@ -578,19 +579,28 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * we have to also clean up the sub-clauses. */ static void -remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) +remove_rel_from_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo, int relid, + int ojrelid) { + Relids new_clause_relids; + /* * initsplan.c is fairly cavalier about allowing RestrictInfos to share * relid sets with other RestrictInfos, and SpecialJoinInfos too. Make * sure this RestrictInfo has its own relid sets before we modify them. - * (In present usage, clause_relids is probably not shared, but - * required_relids could be; let's not assume anything.) + * The 'new_clause_relids' passed to update_clause_relids() must be a + * different instance from the current rinfo->clause_relids, so we make a + * copy of it. + */ + new_clause_relids = bms_copy(rinfo->clause_relids); + new_clause_relids = bms_del_member(new_clause_relids, relid); + new_clause_relids = bms_del_member(new_clause_relids, ojrelid); + update_clause_relids(root, rinfo, new_clause_relids); + + /* + * In present usage, required_relids could be shared, so we make a copy of + * it. */ - rinfo->clause_relids = bms_copy(rinfo->clause_relids); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, relid); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, ojrelid); - /* Likewise for required_relids */ rinfo->required_relids = bms_copy(rinfo->required_relids); rinfo->required_relids = bms_del_member(rinfo->required_relids, relid); rinfo->required_relids = bms_del_member(rinfo->required_relids, ojrelid); @@ -615,14 +625,14 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) { RestrictInfo *rinfo2 = lfirst_node(RestrictInfo, lc2); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } else { RestrictInfo *rinfo2 = castNode(RestrictInfo, orarg); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } } @@ -638,9 +648,11 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) * level(s). */ static void -remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) +remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, + int ojrelid) { ListCell *lc; + int i; /* Fix up the EC's overall relids */ ec->ec_relids = bms_del_member(ec->ec_relids, relid); @@ -667,11 +679,12 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) } /* Fix up the source clauses, in case we can re-use them later */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); } /* @@ -679,7 +692,7 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) * drop them. (At this point, any such clauses would be base restriction * clauses, which we'd not need anymore anyway.) */ - ec->ec_derives = NIL; + ec->ec_derive_indexes = NULL; } /* diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d92d43a17e..28b3a5cabb 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -659,6 +659,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, PlannerInfo *parent_root, root->multiexpr_params = NIL; root->join_domains = NIL; root->eq_classes = NIL; + root->eq_sources = NIL; + root->eq_derives = NIL; root->ec_merging_done = false; root->last_rinfo_serial = 0; root->all_result_relids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index a70404558f..99e1f1ce0d 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1042,6 +1042,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, subroot->multiexpr_params = NIL; subroot->join_domains = NIL; subroot->eq_classes = NIL; + subroot->eq_sources = NIL; + subroot->eq_derives = NIL; subroot->ec_merging_done = false; subroot->last_rinfo_serial = 0; subroot->all_result_relids = NULL; diff --git a/src/backend/optimizer/util/appendinfo.c b/src/backend/optimizer/util/appendinfo.c index 4989722637..4489c2e1bf 100644 --- a/src/backend/optimizer/util/appendinfo.c +++ b/src/backend/optimizer/util/appendinfo.c @@ -495,6 +495,8 @@ adjust_appendrel_attrs_mutator(Node *node, newinfo->right_bucketsize = -1; newinfo->left_mcvfreq = -1; newinfo->right_mcvfreq = -1; + newinfo->eq_sources_index = -1; + newinfo->eq_derives_index = -1; return (Node *) newinfo; } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5b..b5d3984219 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,6 +492,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); + /* + * We do not want to inherit the EquivalenceMember indexes of the parent + * to its child + */ + childrte->eclass_source_indexes = NULL; + childrte->eclass_derive_indexes = NULL; + Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 0b406e9334..59ba503ecb 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -247,6 +247,9 @@ make_restrictinfo_internal(PlannerInfo *root, restrictinfo->left_hasheqoperator = InvalidOid; restrictinfo->right_hasheqoperator = InvalidOid; + restrictinfo->eq_sources_index = -1; + restrictinfo->eq_derives_index = -1; + return restrictinfo; } @@ -403,6 +406,8 @@ commute_restrictinfo(RestrictInfo *rinfo, Oid comm_op) result->right_mcvfreq = rinfo->left_mcvfreq; result->left_hasheqoperator = InvalidOid; result->right_hasheqoperator = InvalidOid; + result->eq_sources_index = -1; + result->eq_derives_index = -1; return result; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 1c314cd907..fefe37b589 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1247,6 +1247,12 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); + Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources + * list for RestrictInfos that mention + * this relation */ + Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives + * list for RestrictInfos that mention + * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index a1f044d8a6..0d06bb150b 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -321,6 +321,12 @@ struct PlannerInfo /* list of active EquivalenceClasses */ List *eq_classes; + /* list of source RestrictInfos used to build EquivalenceClasses */ + List *eq_sources; + + /* list of RestrictInfos derived from EquivalenceClasses */ + List *eq_derives; + /* set true once ECs are canonical */ bool ec_merging_done; @@ -1396,6 +1402,24 @@ typedef struct JoinDomain * its comment for usage. The approach to lookup child members quickly is * described as iterate_child_rel_equivalences() comment. * + * At various locations in the query planner, we must search for source and + * derived RestrictInfos regarding a given EquivalenceClass. For the common + * case, an EquivalenceClass does not have a large number of RestrictInfos, + * however, in cases such as planning queries to partitioned tables, the + * number of members can become large. To maintain planning performance, we + * make use of a bitmap index to allow us to quickly find RestrictInfos in a + * given EquivalenceClass belonging to a given relation or set of relations. + * This is done by storing a list of RestrictInfos belonging to all + * EquivalenceClasses in PlannerInfo and storing a Bitmapset for each + * RelOptInfo which has a bit set for each RestrictInfo in that list which + * relates to the given relation. We also store a Bitmapset to mark all of + * the indexes in the PlannerInfo's list of RestrictInfos in the + * EquivalenceClass. We can quickly find the interesting indexes into the + * PlannerInfo's list by performing a bit-wise AND on the RelOptInfo's + * Bitmapset and the EquivalenceClasses. RestrictInfos must be looked up in + * PlannerInfo by this technique using the ec_source_indexes and + * ec_derive_indexes Bitmapsets. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1414,8 +1438,10 @@ typedef struct EquivalenceClass List *ec_opfamilies; /* btree operator family OIDs */ Oid ec_collation; /* collation, if datatypes are collatable */ List *ec_members; /* list of EquivalenceMembers */ - List *ec_sources; /* list of generating RestrictInfos */ - List *ec_derives; /* list of derived RestrictInfos */ + Bitmapset *ec_source_indexes; /* indexes into PlannerInfo's eq_sources + * list of generating RestrictInfos */ + Bitmapset *ec_derive_indexes; /* indexes into PlannerInfo's eq_derives + * list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except * for child members (see below) */ bool ec_has_const; /* any pseudoconstants in ec_members? */ @@ -2656,7 +2682,12 @@ typedef struct RestrictInfo /* number of base rels in clause_relids */ int num_base_rels pg_node_attr(equal_ignore); - /* The relids (varnos+varnullingrels) actually referenced in the clause: */ + /* + * The relids (varnos+varnullingrels) actually referenced in the clause. + * + * NOTE: This field must be updated through update_clause_relids(), not by + * setting the field directly. + */ Relids clause_relids pg_node_attr(equal_ignore); /* The set of relids required to evaluate the clause: */ @@ -2774,6 +2805,10 @@ typedef struct RestrictInfo /* hash equality operators used for memoize nodes, else InvalidOid */ Oid left_hasheqoperator pg_node_attr(equal_ignore); Oid right_hasheqoperator pg_node_attr(equal_ignore); + + /* the index within root->eq_sources and root->eq_derives */ + int eq_sources_index pg_node_attr(equal_ignore); + int eq_derives_index pg_node_attr(equal_ignore); } RestrictInfo; /* diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 50812e3a5d..a24a5801c5 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -129,6 +129,8 @@ extern Expr *canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation); extern void reconsider_outer_join_clauses(PlannerInfo *root); extern void rebuild_eclass_attr_needed(PlannerInfo *root); +extern void update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids); extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Expr *expr, List *opfamilies, @@ -165,7 +167,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); -extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, +extern RestrictInfo *find_derived_clause_for_ec_member(PlannerInfo *root, + EquivalenceClass *ec, EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, @@ -204,6 +207,22 @@ extern bool eclass_useful_for_merging(PlannerInfo *root, extern bool is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist); extern bool is_redundant_with_indexclauses(RestrictInfo *rinfo, List *indexclauses); +extern Bitmapset *get_ec_source_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_source_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +#ifdef USE_ASSERT_CHECKING +extern void verify_eclass_indexes(PlannerInfo *root, + EquivalenceClass *ec); +#endif /* * pathkeys.c -- 2.45.2.windows.1 [application/octet-stream] v26-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch (14.4K, ../../CAJ2pMkZSqTUMnBMks8B+z4OTQ=V6ohOE=PR7FbNCVJkH3rnsvA@mail.gmail.com/4-v26-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch) download | inline diff: From 6f6058be5bbb03b06c7500a2ced3a17ea0712d18 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 19 Jan 2024 11:43:29 +0900 Subject: [PATCH v26 3/5] Move EquivalenceClass indexes to PlannerInfo In the previous commit, the indexes, namely ec_[source|derive]_indexes, were in RestrictInfo. This was a workaround because RelOptInfo was the best place but some RelOptInfos can be NULL and we cannot store indexes for them. This commit introduces a new struct, EquivalenceClassIndexes. This struct exists in PlannerInfo and holds our indexes. This change prevents a serialization problem with RestirctInfo in the previous commit. --- src/backend/nodes/outfuncs.c | 2 - src/backend/nodes/readfuncs.c | 2 - src/backend/optimizer/path/equivclass.c | 83 ++++++++++++------------- src/backend/optimizer/util/inherit.c | 7 --- src/backend/optimizer/util/relnode.c | 6 ++ src/include/nodes/parsenodes.h | 6 -- src/include/nodes/pathnodes.h | 26 ++++++++ src/tools/pgindent/typedefs.list | 1 + 8 files changed, 74 insertions(+), 59 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index bd6a79b57a..8a635a2a12 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -573,8 +573,6 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); - WRITE_BITMAPSET_FIELD(eclass_source_indexes); - WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 70864fc1b2..be5f19dd7f 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -434,8 +434,6 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_BITMAPSET_FIELD(eclass_source_indexes); - READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 7544ea93f2..ea98e88bf4 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -614,10 +614,10 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, - source_idx); + index->source_indexes = bms_add_member(index->source_indexes, + source_idx); } } @@ -638,10 +638,10 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, - derive_idx); + index->derive_indexes = bms_add_member(index->derive_indexes, + derive_idx); } } @@ -680,22 +680,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(removing_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_del_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_del_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_del_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_del_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -709,22 +709,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(adding_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(!bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_add_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_add_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(!bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_add_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_add_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -739,14 +739,14 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(common_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); + index->source_indexes)); if (rinfo->eq_derives_index != -1) Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); + index->derive_indexes)); } bms_free(common_relids); #endif @@ -3925,9 +3925,9 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + rel_esis = bms_add_members(rel_esis, index->source_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3963,7 +3963,7 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3972,12 +3972,12 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ esis = bms_intersect(ec->ec_source_indexes, - rte->eclass_source_indexes); + index->source_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - esis = bms_int_members(esis, rte->eclass_source_indexes); + index = &root->eclass_indexes_array[i]; + esis = bms_int_members(esis, index->source_indexes); } } @@ -4014,9 +4014,9 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + rel_edis = bms_add_members(rel_edis, index->derive_indexes); } #ifdef USE_ASSERT_CHECKING @@ -4052,7 +4052,7 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -4061,12 +4061,12 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, - rte->eclass_derive_indexes); + index->derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - edis = bms_int_members(edis, rte->eclass_derive_indexes); + index = &root->eclass_indexes_array[i]; + edis = bms_int_members(edis, index->derive_indexes); } } @@ -4089,9 +4089,8 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, /* * verify_eclass_indexes * Verify that there are no missing references between RestrictInfos and - * EquivalenceMember's indexes, namely eclass_source_indexes and - * eclass_derive_indexes. If you modify these indexes, you should check - * them with this function. + * EquivalenceMember's indexes, namely source_indexes and derive_indexes. + * If you modify these indexes, you should check them with this function. */ void verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) @@ -4100,7 +4099,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) /* * All RestrictInfos in root->eq_sources must have references to - * eclass_source_indexes. + * source_indexes. */ foreach(lc, root->eq_sources) { @@ -4117,13 +4116,13 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].source_indexes)); } } /* * All RestrictInfos in root->eq_derives must have references to - * eclass_derive_indexes. + * derive_indexes. */ foreach(lc, root->eq_derives) { @@ -4140,7 +4139,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].derive_indexes)); } } } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index b5d3984219..3c2198ea5b 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,13 +492,6 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); - /* - * We do not want to inherit the EquivalenceMember indexes of the parent - * to its child - */ - childrte->eclass_source_indexes = NULL; - childrte->eclass_derive_indexes = NULL; - Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index f32575b949..c219de44d7 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -119,6 +119,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->simple_rte_array[rti++] = rte; } + root->eclass_indexes_array = (EquivalenceClassIndexes *) + palloc0(size * sizeof(EquivalenceClassIndexes)); + /* append_rel_array is not needed if there are no AppendRelInfos */ if (root->append_rel_list == NIL) { @@ -218,6 +221,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->top_parent_relid_array = NULL; } + root->eclass_indexes_array = + repalloc0_array(root->eclass_indexes_array, EquivalenceClassIndexes, root->simple_rel_array_size, new_size); + root->simple_rel_array_size = new_size; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index fefe37b589..1c314cd907 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1247,12 +1247,6 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); - Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources - * list for RestrictInfos that mention - * this relation */ - Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives - * list for RestrictInfos that mention - * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 0d06bb150b..3eeaa7068b 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -192,6 +192,8 @@ typedef struct PlannerInfo PlannerInfo; #define HAVE_PLANNERINFO_TYPEDEF 1 #endif +struct EquivalenceClassIndexes; + struct PlannerInfo { pg_node_attr(no_copy_equal, no_read, no_query_jumble) @@ -248,6 +250,13 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * eclass_indexes_array is the same length as simple_rel_array and holds + * the indexes of the corresponding rels for faster lookups of + * RestrictInfo. See the EquivalenceClass comment for more details. + */ + struct EquivalenceClassIndexes *eclass_indexes_array pg_node_attr(read_write_ignore); + /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within @@ -1534,6 +1543,23 @@ typedef struct List *ec_members; /* parent and child members */ } EquivalenceChildMemberIterator; +/* + * EquivalenceClassIndexes + * + * As mentioned in the EquivalenceClass comment, we introduce a + * bitmapset-based indexing mechanism for faster lookups of RestrictInfo. This + * struct exists for each relation and holds the corresponding indexes. + */ +typedef struct EquivalenceClassIndexes +{ + Bitmapset *source_indexes; /* Indexes in PlannerInfo's eq_sources list + * for RestrictInfos that mention this + * relation */ + Bitmapset *derive_indexes; /* Indexes in PlannerInfo's eq_derives list + * for RestrictInfos that mention this + * relation */ +} EquivalenceClassIndexes; + /* * PathKeys * diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 623a770b0e..1170c0a891 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -685,6 +685,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceChildMemberIterator EquivalenceClass +EquivalenceClassIndexes EquivalenceMember ErrorContextCallback ErrorData -- 2.45.2.windows.1 [application/octet-stream] v26-0004-Rename-add_eq_member-to-add_parent_eq_member.patch (5.2K, ../../CAJ2pMkZSqTUMnBMks8B+z4OTQ=V6ohOE=PR7FbNCVJkH3rnsvA@mail.gmail.com/5-v26-0004-Rename-add_eq_member-to-add_parent_eq_member.patch) download | inline diff: From 97c87e1dcf3977c312e7edd7c2c8b915a02aed96 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Wed, 21 Aug 2024 13:54:59 +0900 Subject: [PATCH v26 4/5] Rename add_eq_member() to add_parent_eq_member() After our changes, add_eq_member() no longer creates and adds child EquivalenceMembers. This commit renames it to add_parent_eq_member() to clarify that it only creates parent members, and that we need to use make_eq_member() to handle child EquivalenceMembers. --- src/backend/optimizer/path/equivclass.c | 54 ++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index ea98e88bf4..1cb1bfa075 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -38,10 +38,10 @@ static EquivalenceMember *make_eq_member(EquivalenceClass *ec, JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype); -static EquivalenceMember *add_eq_member(EquivalenceClass *ec, - Expr *expr, Relids relids, - JoinDomain *jdomain, - Oid datatype); +static EquivalenceMember *add_parent_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + Oid datatype); static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo); static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, @@ -390,8 +390,8 @@ process_equivalence(PlannerInfo *root, else if (ec1) { /* Case 3: add item2 to ec1 */ - em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, item2_type); + em2 = add_parent_eq_member(ec1, item2, item2_relids, + jdomain, item2_type); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -408,8 +408,8 @@ process_equivalence(PlannerInfo *root, else if (ec2) { /* Case 3: add item1 to ec2 */ - em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, item1_type); + em1 = add_parent_eq_member(ec2, item1, item1_relids, + jdomain, item1_type); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -441,10 +441,10 @@ process_equivalence(PlannerInfo *root, ec->ec_min_security = restrictinfo->security_level; ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; - em1 = add_eq_member(ec, item1, item1_relids, - jdomain, item1_type); - em2 = add_eq_member(ec, item2, item2_relids, - jdomain, item2_type); + em1 = add_parent_eq_member(ec, item1, item1_relids, + jdomain, item1_type); + em2 = add_parent_eq_member(ec, item2, item2_relids, + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -579,7 +579,7 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, } /* - * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * add_parent_eq_member - build a new parent EquivalenceMember and add it to an EC * * Note: We don't have a function to add a child member like * add_child_eq_member() because how to do it depends on the relations they are @@ -587,8 +587,8 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, * add_child_join_rel_equivalences() to see how to add child members. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, Oid datatype) +add_parent_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) { EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, NULL, datatype); @@ -922,14 +922,14 @@ get_eclass_for_sort_expr(PlannerInfo *root, */ expr_relids = pull_varnos(root, (Node *) expr); - newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, opcintype); + newem = add_parent_eq_member(newec, copyObject(expr), expr_relids, + jdomain, opcintype); /* - * add_eq_member doesn't check for volatile functions, set-returning - * functions, aggregates, or window functions, but such could appear in - * sort expressions; so we have to check whether its const-marking was - * correct. + * add_parent_eq_member doesn't check for volatile functions, + * set-returning functions, aggregates, or window functions, but such + * could appear in sort expressions; so we have to check whether its + * const-marking was correct. */ if (newec->ec_has_const) { @@ -3272,12 +3272,12 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + add_parent_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); lc2 = lnext(setop_pathkeys, lc2); } -- 2.45.2.windows.1 [application/octet-stream] v26-0005-Resolve-conflict-with-commit-66c0185.patch (6.4K, ../../CAJ2pMkZSqTUMnBMks8B+z4OTQ=V6ohOE=PR7FbNCVJkH3rnsvA@mail.gmail.com/6-v26-0005-Resolve-conflict-with-commit-66c0185.patch) download | inline diff: From 6837c1e592e0870043b64e2e9046f587073dd2f5 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Tue, 27 Aug 2024 13:20:29 +0900 Subject: [PATCH v26 5/5] Resolve conflict with commit 66c0185 This commit resolves a conflict with 66c0185, which added add_setop_child_rel_equivalences(). The function creates child EquivalenceMembers to efficiently implement UNION queries. This commit adjusts our optimization to handle this type of child EquivalenceMembers based on UNION parent-child relationships. --- src/backend/optimizer/path/equivclass.c | 51 ++++++++++++++++++++++--- src/backend/optimizer/util/inherit.c | 8 +++- src/backend/optimizer/util/relnode.c | 10 +++-- src/include/nodes/pathnodes.h | 2 +- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 1cb1bfa075..2ff9c606d5 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -3255,7 +3255,9 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, { TargetEntry *tle = lfirst_node(TargetEntry, lc); EquivalenceMember *parent_em; + EquivalenceMember *child_em; PathKey *pk; + Index parent_relid; if (tle->resjunk) continue; @@ -3272,12 +3274,49 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_parent_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + child_em = make_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); + child_rel->eclass_child_members = + lappend(child_rel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated using + * 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + parent_em->em_child_relids = + bms_add_member(parent_em->em_child_relids, child_rel->relid); + + /* + * Make an UNION parent-child relationship between parent_em and + * child_rel->relid. We record this relationship in + * root->top_parent_relid_array, which generally has AppendRelInfo + * relationships. We use the same array here to retrieve UNION child + * members. + * + * XXX Here we treat the first member of parent_em->em_relids as a + * parent of child_rel. Is this correct? What happens if + * parent_em->em_relids has two or more members? + */ + parent_relid = bms_next_member(parent_em->em_relids, -1); + if (root->top_parent_relid_array == NULL) + { + /* + * If the array is NULL, allocate it here. + */ + root->top_parent_relid_array = (Index *) + palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[child_rel->relid] == -1 || + root->top_parent_relid_array[child_rel->relid] == parent_relid); + root->top_parent_relid_array[child_rel->relid] = parent_relid; lc2 = lnext(setop_pathkeys, lc2); } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5b..7b2390687e 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -582,9 +582,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, * Find a top parent rel's index and save it to top_parent_relid_array. */ if (root->top_parent_relid_array == NULL) + { root->top_parent_relid_array = - (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); - Assert(root->top_parent_relid_array[childRTindex] == 0); + (Index *) palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[childRTindex] == -1); topParentRTindex = parentRTindex; while (root->append_rel_array[topParentRTindex] != NULL && root->append_rel_array[topParentRTindex]->parent_relid != 0) diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index c219de44d7..5ea9a41008 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -133,7 +133,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); root->top_parent_relid_array = (Index *) - palloc0(size * sizeof(Index)); + palloc(size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * size); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -206,7 +208,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); root->top_parent_relid_array = - repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + repalloc_array(root->top_parent_relid_array, Index, new_size); + MemSet(root->top_parent_relid_array + root->simple_rel_array_size, -1, + sizeof(Index) * add_size); } else { @@ -1608,7 +1612,7 @@ find_relids_top_parents_slow(PlannerInfo *root, Relids relids) { int top_parent_relid = (int) top_parent_relid_array[i]; - if (top_parent_relid == 0) + if (top_parent_relid == -1) top_parent_relid = i; /* 'i' has no parents, so add itself */ else if (top_parent_relid != i) is_top_parent = false; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3eeaa7068b..8dc32eb606 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -260,7 +260,7 @@ struct PlannerInfo /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within - * simple_rel_array. The element can be zero if the rel has no parents, + * simple_rel_array. The element can be -1 if the rel has no parents, * i.e., is itself in a top-level. */ Index *top_parent_relid_array pg_node_attr(read_write_ignore); -- 2.45.2.windows.1 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-10-15 03:20 Yuya Watari <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-10-15 03:20 UTC (permalink / raw) To: PostgreSQL Developers <[email protected]>; +Cc: jian he <[email protected]>; Ashutosh Bapat <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> Hello, On Tue, Oct 1, 2024 at 11:35 AM Yuya Watari <[email protected]> wrote: > > I noticed the patches do not apply to the current master. I have > attached the rebased version. There are no changes besides the rebase. The previous patches do not apply to the current master, so I have attached the rebased version. -- Best regards, Yuya Watari Attachments: [application/octet-stream] v27-0001-Speed-up-searches-for-child-EquivalenceMembers.patch (59.2K, ../../CAJ2pMkbOoQw0_Kih7Bn3ENgHjLqNmAO-DqG2NUaRyhAEfTMy7Q@mail.gmail.com/2-v27-0001-Speed-up-searches-for-child-EquivalenceMembers.patch) download | inline diff: From caac27cf35d50107e3248bdceffcb6702d7e917d Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:43:36 +0900 Subject: [PATCH v27 1/5] Speed up searches for child EquivalenceMembers Traditionally, child EquivalenceMembers were in the EquivalenceClass->ec_members. When we wanted to find some child members matching a request, we had to perform a linear search. This search became heavy when tables had many partitions, leading to much planning time. After this commit, child EquivalenceMembers no longer exist in ec_members. Instead, RelOptInfos have them. This change demonstrates a significant performance improvement in planning time. --- contrib/postgres_fdw/postgres_fdw.c | 22 +- src/backend/optimizer/path/equivclass.c | 414 ++++++++++++++++++++---- src/backend/optimizer/path/indxpath.c | 35 +- src/backend/optimizer/path/pathkeys.c | 9 +- src/backend/optimizer/plan/createplan.c | 61 ++-- src/backend/optimizer/util/inherit.c | 14 + src/backend/optimizer/util/relnode.c | 89 +++++ src/include/nodes/pathnodes.h | 61 ++++ src/include/optimizer/pathnode.h | 18 ++ src/include/optimizer/paths.h | 12 +- src/tools/pgindent/typedefs.list | 1 + 11 files changed, 622 insertions(+), 114 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index adc62576d1..93cbc53f03 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7831,13 +7831,21 @@ conversion_error_callback(void *arg) EquivalenceMember * find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) { - ListCell *lc; - PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + Relids top_parent_rel_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, rel->relids); /* * Note we require !bms_is_empty, else we'd accept constant @@ -7849,6 +7857,7 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) is_foreign_expr(root, rel, em->em_expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -7902,9 +7911,8 @@ find_em_for_rel_target(PlannerInfo *root, EquivalenceClass *ec, if (em->em_is_const) continue; - /* Ignore child members */ - if (em->em_is_child) - continue; + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* Match if same expression (after stripping relabel) */ em_expr = em->em_expr; diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index fae137dd82..bcd56da3d2 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -33,10 +33,14 @@ #include "utils/lsyscache.h" +static EquivalenceMember *make_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + EquivalenceMember *parent, + Oid datatype); static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, - EquivalenceMember *parent, Oid datatype); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); @@ -68,6 +72,11 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo); static JoinDomain *find_join_domain(PlannerInfo *root, Relids relids); +static void add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel); static Bitmapset *get_eclass_indexes_for_relids(PlannerInfo *root, Relids relids); static Bitmapset *get_common_eclass_indexes(PlannerInfo *root, Relids relids1, @@ -373,7 +382,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); @@ -390,7 +399,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); @@ -422,9 +431,9 @@ process_equivalence(PlannerInfo *root, ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; em1 = add_eq_member(ec, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); em2 = add_eq_member(ec, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -510,11 +519,16 @@ canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation) } /* - * add_eq_member - build a new EquivalenceMember and add it to an EC + * make_eq_member + * + * Build a new EquivalenceMember without adding it to an EC. If 'parent' + * parameter is NULL, the result will be a parent member, otherwise a child + * member. Note that child EquivalenceMembers should not be added to its + * parent EquivalenceClass. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) +make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) { EquivalenceMember *em = makeNode(EquivalenceMember); @@ -525,6 +539,8 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, em->em_datatype = datatype; em->em_jdomain = jdomain; em->em_parent = parent; + em->em_child_relids = NULL; + em->em_child_joinrel_relids = NULL; if (bms_is_empty(relids)) { @@ -545,11 +561,29 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, { ec->ec_relids = bms_add_members(ec->ec_relids, relids); } - ec->ec_members = lappend(ec->ec_members, em); return em; } +/* + * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * + * Note: We don't have a function to add a child member like + * add_child_eq_member() because how to do it depends on the relations they are + * translated from. See add_child_rel_equivalences() and + * add_child_join_rel_equivalences() to see how to add child members. + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + + ec->ec_members = lappend(ec->ec_members, em); + return em; +} + /* * get_eclass_for_sort_expr @@ -598,6 +632,17 @@ get_eclass_for_sort_expr(PlannerInfo *root, EquivalenceMember *newem; ListCell *lc1; MemoryContext oldcontext; + Relids top_parent_rel; + + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_rel = find_relids_top_parents(root, rel); /* * Ensure the expression exposes the correct type and collation. @@ -616,7 +661,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; /* * Never match to a volatile EC, except when we are looking at another @@ -631,16 +677,28 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (!equal(opfamilies, cur_ec->ec_opfamilies)) continue; - foreach(lc2, cur_ec->ec_members) + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel); /* - * Ignore child members unless they match the request. + * Ignore child members unless they match the request. If this + * EquivalenceMember is a child, i.e., translated above, it should + * match the request. We cannot assert this if a request is + * bms_is_subset(). */ - if (cur_em->em_is_child && - !bms_equal(cur_em->em_relids, rel)) - continue; + Assert(!cur_em->em_is_child || bms_equal(cur_em->em_relids, rel)); /* * Match constants only within the same JoinDomain (see @@ -653,6 +711,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, equal(expr, cur_em->em_expr)) return cur_ec; /* Match! */ } + dispose_eclass_child_member_iterator(&it); } /* No match; does caller want a NULL result? */ @@ -690,7 +749,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, expr_relids = pull_varnos(root, (Node *) expr); newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, NULL, opcintype); + jdomain, opcintype); /* * add_eq_member doesn't check for volatile functions, set-returning @@ -760,19 +819,25 @@ get_eclass_for_sort_expr(PlannerInfo *root, * Child EC members are ignored unless they belong to given 'relids'. */ EquivalenceMember * -find_ec_member_matching_expr(EquivalenceClass *ec, +find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, Expr *expr, Relids relids) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); /* We ignore binary-compatible relabeling on both ends */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); Expr *emexpr; /* @@ -782,6 +847,11 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -799,6 +869,7 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (equal(emexpr, expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -841,7 +912,9 @@ find_computable_ec_member(PlannerInfo *root, bool require_parallel_safe) { List *exprvars; - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; /* * Pull out the Vars and quasi-Vars present in "exprs". In the typical @@ -854,9 +927,13 @@ find_computable_ec_member(PlannerInfo *root, PVC_INCLUDE_WINDOWFUNCS | PVC_INCLUDE_PLACEHOLDERS); - foreach(lc, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_relids = find_relids_top_parents(root, relids); + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); List *emvars; ListCell *lc2; @@ -867,6 +944,11 @@ find_computable_ec_member(PlannerInfo *root, if (em->em_is_const) continue; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -900,6 +982,7 @@ find_computable_ec_member(PlannerInfo *root, return em; /* found usable expression */ } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -938,7 +1021,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, { Expr *targetexpr = (Expr *) lfirst(lc); - em = find_ec_member_matching_expr(ec, targetexpr, rel->relids); + em = find_ec_member_matching_expr(root, ec, targetexpr, rel->relids); if (!em) continue; @@ -1563,7 +1646,12 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *new_members = NIL; List *outer_members = NIL; List *inner_members = NIL; - ListCell *lc1; + Relids top_parent_join_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; + + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_join_relids = find_relids_top_parents(root, join_relids); /* * First, scan the EC to identify member values that are computable at the @@ -1574,9 +1662,14 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * as well as to at least one input member, plus enforce at least one * outer-rel member equal to at least one inner-rel member. */ - foreach(lc1, ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + bms_is_subset(cur_em->em_relids, top_parent_join_relids)) + iterate_child_rel_equivalences(&it, root, ec, cur_em, join_relids); /* * We don't need to check explicitly for child EC members. This test @@ -1593,6 +1686,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, else new_members = lappend(new_members, cur_em); } + dispose_eclass_child_member_iterator(&it); /* * First, select the joinclause if needed. We can equate any one outer @@ -1610,6 +1704,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, Oid best_eq_op = InvalidOid; int best_score = -1; RestrictInfo *rinfo; + ListCell *lc1; foreach(lc1, outer_members) { @@ -1684,6 +1779,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *old_members = list_concat(outer_members, inner_members); EquivalenceMember *prev_em = NULL; RestrictInfo *rinfo; + ListCell *lc1; /* For now, arbitrarily take the first old_member as the one to use */ if (old_members) @@ -1691,7 +1787,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, foreach(lc1, new_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + cur_em = (EquivalenceMember *) lfirst(lc1); if (prev_em != NULL) { @@ -2404,6 +2500,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) if (matchleft && matchright) { cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); + Assert(!bms_is_empty(coal_em->em_relids)); return true; } @@ -2525,8 +2622,8 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, Oid opfamily) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); if (equal(item1, em->em_expr)) item1member = true; else if (equal(item2, em->em_expr)) @@ -2597,8 +2694,8 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); Var *var; - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* EM must be a Var, possibly with RelabelType */ var = (Var *) em->em_expr; @@ -2695,6 +2792,7 @@ add_child_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_rel->top_parent_relids; Relids child_relids = child_rel->relids; int i; + ListCell *lc; /* * EC merging should be complete already, so we can use the parent rel's @@ -2707,7 +2805,6 @@ add_child_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2720,15 +2817,9 @@ add_child_rel_equivalences(PlannerInfo *root, /* Sanity check eclass_indexes only contain ECs for parent_rel */ Assert(bms_is_subset(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2741,8 +2832,8 @@ add_child_rel_equivalences(PlannerInfo *root, * combinations of children. (But add_child_join_rel_equivalences * may add targeted combinations for partitionwise-join purposes.) */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * Consider only members that reference and can be computed at @@ -2758,6 +2849,7 @@ add_child_rel_equivalences(PlannerInfo *root, /* OK, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_rel->reloptkind == RELOPT_BASEREL) { @@ -2787,9 +2879,20 @@ add_child_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, + child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_relids = bms_add_member(cur_em->em_child_relids, + child_rel->relid); /* Record this EC index for the child rel */ child_rel->eclass_indexes = bms_add_member(child_rel->eclass_indexes, i); @@ -2818,6 +2921,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; MemoryContext oldcontext; + ListCell *lc; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2839,7 +2943,6 @@ add_child_join_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(matching_ecs, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2852,15 +2955,9 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Sanity check on get_eclass_indexes_for_relids result */ Assert(bms_overlap(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2869,8 +2966,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, * We consider only original EC members here, not * already-transformed child members. */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * We may ignore expressions that reference a single baserel, @@ -2885,6 +2982,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Yes, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_joinrel->reloptkind == RELOPT_JOINREL) { @@ -2915,9 +3013,21 @@ add_child_join_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_joinrel->eclass_child_members = + lappend(child_joinrel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_joinrel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_joinrel_relids = + bms_add_member(cur_em->em_child_joinrel_relids, + child_joinrel->join_rel_list_index); } } } @@ -2986,6 +3096,161 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, list_length(root->eq_classes) - 1); } +/* + * setup_eclass_child_member_iterator + * Setup an EquivalenceChildMemberIterator 'it' so that it can iterate over + * EquivalenceMembers in 'ec'. + * + * Once used, the caller should dispose of the iterator by calling + * dispose_eclass_child_member_iterator(). + */ +void +setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec) +{ + it->index = -1; + it->modified = false; + it->ec_members = ec->ec_members; +} + +/* + * eclass_child_member_iterator_next + * Get a next EquivalenceMember from an EquivalenceChildMemberIterator 'it' + * that was setup by setup_eclass_child_member_iterator(). NULL is returned + * if there are no members left. + */ +EquivalenceMember * +eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it) +{ + if (++it->index < list_length(it->ec_members)) + return list_nth_node(EquivalenceMember, it->ec_members, it->index); + return NULL; +} + +/* + * dispose_eclass_child_member_iterator + * Free any memory allocated by the iterator. + */ +void +dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it) +{ + /* + * XXX Should we use list_free()? I decided to use this style to take + * advantage of speculative execution. + */ + if (unlikely(it->modified)) + pfree(it->ec_members); +} + +/* + * add_transformed_child_version + * Add a transformed EquivalenceMember referencing the given child_rel to + * the iterator. + * + * This function is expected to be called only from + * iterate_child_rel_equivalences(). + */ +static void +add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel) +{ + ListCell *lc; + + foreach(lc, child_rel->eclass_child_members) + { + EquivalenceMember *child_em = lfirst_node(EquivalenceMember, lc); + + /* Skip unwanted EquivalenceMembers */ + if (child_em->em_parent != parent_em) + continue; + + /* + * If this is the first time the iterator's list has been modified, we + * need to make a copy of it. + */ + if (!it->modified) + { + it->ec_members = list_copy(it->ec_members); + it->modified = true; + } + + /* Add this child EquivalenceMember to the list */ + it->ec_members = lappend(it->ec_members, child_em); + } +} + +/* + * iterate_child_rel_equivalences + * Add transformed EquivalenceMembers referencing child rels in the given + * child_relids to the iterator. + * + * The transformation is done in add_transformed_child_version(). + */ +void +iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids) +{ + int i; + Relids matching_relids; + + /* The given EquivalenceMember should be parent */ + Assert(!parent_em->em_is_child); + + /* + * EquivalenceClass->ec_members has only parent EquivalenceMembers. Child + * members are translated using child RelOptInfos and stored in them. This + * is done in add_child_rel_equivalences() and + * add_child_join_rel_equivalences(). To retrieve child EquivalenceMembers + * of some parent, we need to know which RelOptInfos have such child + * members. This information is stored in indexes named em_child_relids + * and em_child_joinrel_relids. + * + * This function iterates over the child EquivalenceMembers that reference + * the given 'child_relids'. To do this, we first intersect 'child_relids' + * with these indexes. The result contains Relids of RelOptInfos that have + * child EquivalenceMembers we want to retrieve. Then we get the child + * members from the RelOptInfos using add_transformed_child_version(). + * + * We need to do these steps for each of the two indexes. + */ + + /* + * First, we translate simple rels. + */ + matching_relids = bms_intersect(parent_em->em_child_relids, + child_relids); + i = -1; + while ((i = bms_next_member(matching_relids, i)) >= 0) + { + /* Add transformed child version for this child rel */ + RelOptInfo *child_rel = root->simple_rel_array[i]; + + Assert(child_rel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_rel); + } + bms_free(matching_relids); + + /* + * Next, we try to translate join rels. + */ + i = -1; + while ((i = bms_next_member(parent_em->em_child_joinrel_relids, i)) >= 0) + { + /* Add transformed child version for this child join rel */ + RelOptInfo *child_joinrel = + list_nth_node(RelOptInfo, root->join_rel_list, i); + + Assert(child_joinrel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_joinrel); + } +} + /* * generate_implied_equalities_for_column @@ -3019,7 +3284,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, { List *result = NIL; bool is_child_rel = (rel->reloptkind == RELOPT_OTHER_MEMBER_REL); - Relids parent_relids; + Relids parent_relids, + top_parent_rel_relids; int i; /* Should be OK to rely on eclass_indexes */ @@ -3028,6 +3294,9 @@ generate_implied_equalities_for_column(PlannerInfo *root, /* Indexes are available only on base or "other" member relations. */ Assert(IS_SIMPLE_REL(rel)); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + /* If it's a child rel, we'll need to know what its parent(s) are */ if (is_child_rel) parent_relids = find_childrel_parents(root, rel); @@ -3040,6 +3309,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); EquivalenceMember *cur_em; ListCell *lc2; + EquivalenceChildMemberIterator it; /* Sanity check eclass_indexes only contain ECs for rel */ Assert(is_child_rel || bms_is_subset(rel->relids, cur_ec->ec_relids)); @@ -3061,15 +3331,19 @@ generate_implied_equalities_for_column(PlannerInfo *root, * corner cases, so for now we live with just reporting the first * match. See also get_eclass_for_sort_expr.) */ - cur_em = NULL; - foreach(lc2, cur_ec->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - cur_em = (EquivalenceMember *) lfirst(lc2); + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel->relids); if (bms_equal(cur_em->em_relids, rel->relids) && callback(root, rel, cur_ec, cur_em, callback_arg)) break; - cur_em = NULL; } + dispose_eclass_child_member_iterator(&it); if (!cur_em) continue; @@ -3084,8 +3358,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, Oid eq_op; RestrictInfo *rinfo; - if (other_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!other_em->em_is_child); /* Make sure it'll be a join to a different rel */ if (other_em == cur_em || @@ -3303,8 +3577,8 @@ eclass_useful_for_merging(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); if (!bms_overlap(cur_em->em_relids, relids)) return true; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index c0fcc7d78d..ce14ca324f 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -183,7 +183,7 @@ static IndexClause *expand_indexqual_rowcompare(PlannerInfo *root, IndexOptInfo *index, Oid expr_op, bool var_on_left); -static void match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +static void match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p); static Expr *match_clause_to_ordering_op(IndexOptInfo *index, @@ -927,7 +927,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * query_pathkeys will allow an incremental sort to be considered on * the index's partially sorted results. */ - match_pathkeys_to_index(index, root->query_pathkeys, + match_pathkeys_to_index(root, index, root->query_pathkeys, &orderbyclauses, &orderbyclausecols); if (list_length(root->query_pathkeys) == list_length(orderbyclauses)) @@ -3017,12 +3017,16 @@ expand_indexqual_rowcompare(PlannerInfo *root, * item in the given 'pathkeys' list. */ static void -match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p) { + Relids top_parent_index_relids; ListCell *lc1; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_index_relids = find_relids_top_parents(root, index->rel->relids); + *orderby_clauses_p = NIL; /* set default results */ *clause_columns_p = NIL; @@ -3034,7 +3038,8 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, { PathKey *pathkey = (PathKey *) lfirst(lc1); bool found = false; - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *member; /* Pathkey must request default sort order for the target opfamily */ @@ -3054,15 +3059,30 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, * be considered to match more than one pathkey list, which is OK * here. See also get_eclass_for_sort_expr.) */ - foreach(lc2, pathkey->pk_eclass->ec_members) + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + setup_eclass_child_member_iterator(&it, pathkey->pk_eclass); + while ((member = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *member = (EquivalenceMember *) lfirst(lc2); int indexcol; + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + bms_equal(member->em_relids, top_parent_index_relids)) + iterate_child_rel_equivalences(&it, root, pathkey->pk_eclass, member, + index->rel->relids); + /* No possibility of match if it references other relations */ - if (!bms_equal(member->em_relids, index->rel->relids)) + if (!member->em_is_child && + !bms_equal(member->em_relids, index->rel->relids)) continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(bms_equal(member->em_relids, index->rel->relids)); + /* * We allow any column of the index to match each pathkey; they * don't have to match left-to-right as you might expect. This is @@ -3091,6 +3111,7 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, if (found) /* don't want to look at remaining members */ break; } + dispose_eclass_child_member_iterator(&it); /* * Return the matches found so far when this pathkey couldn't be diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 7cf15e89b6..b6410f31da 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -1151,8 +1151,8 @@ convert_subquery_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid sub_expr_coll = sub_eclass->ec_collation; ListCell *k; - if (sub_member->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!sub_member->em_is_child); foreach(k, subquery_tlist) { @@ -1709,8 +1709,11 @@ select_outer_pathkeys_for_merge(PlannerInfo *root, { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); + /* Potential future join partner? */ - if (!em->em_is_const && !em->em_is_child && + if (!em->em_is_const && !bms_overlap(em->em_relids, joinrel->relids)) score++; } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index f2ed0d81f6..cfb9dd75fe 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -264,7 +264,9 @@ static IncrementalSort *make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); -static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -273,9 +275,11 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); -static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Sort *make_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids); -static IncrementalSort *make_incrementalsort_from_pathkeys(Plan *lefttree, +static IncrementalSort *make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, int nPresortedCols); static Sort *make_sort_from_groupcols(List *groupcls, AttrNumber *grpColIdx, @@ -297,7 +301,7 @@ static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); static Unique *make_unique_from_sortclauses(Plan *lefttree, List *distinctList); -static Unique *make_unique_from_pathkeys(Plan *lefttree, +static Unique *make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, int numCols); static Gather *make_gather(List *qptlist, List *qpqual, int nworkers, int rescan_param, bool single_copy, Plan *subplan); @@ -1285,7 +1289,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * function result; it must be the same plan node. However, we then * need to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys((Plan *) plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, (Plan *) plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1329,7 +1333,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * don't need an explicit sort, to make sure they are returning * the same sort key columns the Append expects. */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, nodeSortColIdx, false, @@ -1470,7 +1474,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * function result; it must be the same plan node. However, we then need * to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys(plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1501,7 +1505,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, subplan = create_plan_recurse(root, subpath, CP_EXACT_TLIST); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, node->sortColIdx, false, @@ -1981,7 +1985,7 @@ create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) Assert(pathkeys != NIL); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, best_path->subpath->parent->relids, gm_plan->sortColIdx, false, @@ -2195,7 +2199,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) * relids. Thus, if this sort path is based on a child relation, we must * pass its relids. */ - plan = make_sort_from_pathkeys(subplan, best_path->path.pathkeys, + plan = make_sort_from_pathkeys(root, subplan, best_path->path.pathkeys, IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -2219,7 +2223,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, /* See comments in create_sort_plan() above */ subplan = create_plan_recurse(root, best_path->spath.subpath, flags | CP_SMALL_TLIST); - plan = make_incrementalsort_from_pathkeys(subplan, + plan = make_incrementalsort_from_pathkeys(root, subplan, best_path->spath.path.pathkeys, IS_OTHER_REL(best_path->spath.subpath->parent) ? best_path->spath.path.parent->relids : NULL, @@ -2288,7 +2292,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag subplan = create_plan_recurse(root, best_path->subpath, flags | CP_LABEL_TLIST); - plan = make_unique_from_pathkeys(subplan, + plan = make_unique_from_pathkeys(root, subplan, best_path->path.pathkeys, best_path->numkeys); @@ -4550,7 +4554,8 @@ create_mergejoin_plan(PlannerInfo *root, if (!use_incremental_sort) { sort_plan = (Plan *) - make_sort_from_pathkeys(outer_plan, + make_sort_from_pathkeys(root, + outer_plan, best_path->outersortkeys, outer_relids); @@ -4559,7 +4564,8 @@ create_mergejoin_plan(PlannerInfo *root, else { sort_plan = (Plan *) - make_incrementalsort_from_pathkeys(outer_plan, + make_incrementalsort_from_pathkeys(root, + outer_plan, best_path->outersortkeys, outer_relids, presorted_keys); @@ -4584,7 +4590,7 @@ create_mergejoin_plan(PlannerInfo *root, */ Relids inner_relids = inner_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(inner_plan, + Sort *sort = make_sort_from_pathkeys(root, inner_plan, best_path->innersortkeys, inner_relids); @@ -6236,7 +6242,7 @@ make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, * or a Result stacked atop lefttree). */ static Plan * -prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -6303,7 +6309,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ @@ -6331,7 +6337,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr already in tlist */ @@ -6347,7 +6353,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, /* * No matching tlist item; look for a computable expression. */ - em = find_computable_ec_member(NULL, ec, tlist, relids, false); + em = find_computable_ec_member(root, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; @@ -6418,7 +6424,8 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ static Sort * -make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) +make_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + Relids relids) { int numsortkeys; AttrNumber *sortColIdx; @@ -6427,7 +6434,7 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6453,8 +6460,9 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) * 'nPresortedCols' is the number of presorted columns in input tuples */ static IncrementalSort * -make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, - Relids relids, int nPresortedCols) +make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, + List *pathkeys, Relids relids, + int nPresortedCols) { int numsortkeys; AttrNumber *sortColIdx; @@ -6463,7 +6471,7 @@ make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6822,7 +6830,8 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) * as above, but use pathkeys to identify the sort columns and semantics */ static Unique * -make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) +make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + int numCols) { Unique *node = makeNode(Unique); Plan *plan = &node->plan; @@ -6885,7 +6894,7 @@ make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) foreach(j, plan->targetlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, NULL); + em = find_ec_member_matching_expr(root, ec, tle->expr, NULL); if (em) { /* found expr already in tlist */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index c5b906a9d4..3c2198ea5b 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -470,6 +470,7 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, RelationGetRelid(parentrel); Oid childOID = RelationGetRelid(childrel); RangeTblEntry *childrte; + Index topParentRTindex; Index childRTindex; AppendRelInfo *appinfo; TupleDesc child_tupdesc; @@ -577,6 +578,19 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, Assert(root->append_rel_array[childRTindex] == NULL); root->append_rel_array[childRTindex] = appinfo; + /* + * Find a top parent rel's index and save it to top_parent_relid_array. + */ + if (root->top_parent_relid_array == NULL) + root->top_parent_relid_array = + (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); + Assert(root->top_parent_relid_array[childRTindex] == 0); + topParentRTindex = parentRTindex; + while (root->append_rel_array[topParentRTindex] != NULL && + root->append_rel_array[topParentRTindex]->parent_relid != 0) + topParentRTindex = root->append_rel_array[topParentRTindex]->parent_relid; + root->top_parent_relid_array[childRTindex] = topParentRTindex; + /* * Build a PlanRowMark if parent is marked FOR UPDATE/SHARE. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index d7266e4cdb..f32575b949 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -123,11 +123,14 @@ setup_simple_rel_arrays(PlannerInfo *root) if (root->append_rel_list == NIL) { root->append_rel_array = NULL; + root->top_parent_relid_array = NULL; return; } root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); + root->top_parent_relid_array = (Index *) + palloc0(size * sizeof(Index)); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -148,6 +151,27 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array[child_relid] = appinfo; } + + /* + * Find a top parent rel's relid for each AppendRelInfo. We cannot do this + * in the last foreach loop because there may be multi-level parent-child + * relations. + */ + for (int i = 0; i < size; i++) + { + int top_parent_relid; + + if (root->append_rel_array[i] == NULL) + continue; + + top_parent_relid = root->append_rel_array[i]->parent_relid; + + while (root->append_rel_array[top_parent_relid] != NULL && + root->append_rel_array[top_parent_relid]->parent_relid != 0) + top_parent_relid = root->append_rel_array[top_parent_relid]->parent_relid; + + root->top_parent_relid_array[i] = top_parent_relid; + } } /* @@ -175,12 +199,25 @@ expand_planner_arrays(PlannerInfo *root, int add_size) repalloc0_array(root->simple_rte_array, RangeTblEntry *, root->simple_rel_array_size, new_size); if (root->append_rel_array) + { root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); + root->top_parent_relid_array = + repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + } else + { root->append_rel_array = palloc0_array(AppendRelInfo *, new_size); + /* + * We do not allocate top_parent_relid_array here so that + * find_relids_top_parents() can early find all of the given Relids + * are top-level. + */ + root->top_parent_relid_array = NULL; + } + root->simple_rel_array_size = new_size; } @@ -234,6 +271,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; + rel->eclass_child_members = NIL; rel->serverid = InvalidOid; if (rte->rtekind == RTE_RELATION) { @@ -629,6 +667,12 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) /* GEQO requires us to append the new joinrel to the end of the list! */ root->join_rel_list = lappend(root->join_rel_list, joinrel); + /* + * Store the index of this joinrel to use in + * add_child_join_rel_equivalences(). + */ + joinrel->join_rel_list_index = list_length(root->join_rel_list) - 1; + /* store it into the auxiliary hashtable if there is one. */ if (root->join_rel_hash) { @@ -741,6 +785,7 @@ build_join_rel(PlannerInfo *root, joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -928,6 +973,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -1490,6 +1536,7 @@ fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids) upperrel->cheapest_total_path = NULL; upperrel->cheapest_unique_path = NULL; upperrel->cheapest_parameterized_paths = NIL; + upperrel->eclass_child_members = NIL; /* XXX Is this required? */ root->upper_rels[kind] = lappend(root->upper_rels[kind], upperrel); @@ -1530,6 +1577,48 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } +/* + * find_relids_top_parents_slow + * Slow path of find_relids_top_parents() macro. + * + * Do not call this directly; use the macro instead. See the macro comment for + * more details. + */ +Relids +find_relids_top_parents_slow(PlannerInfo *root, Relids relids) +{ + Index *top_parent_relid_array = root->top_parent_relid_array; + Relids result; + bool is_top_parent; + int i; + + /* This function should be called in the slow path */ + Assert(top_parent_relid_array != NULL); + + result = NULL; + is_top_parent = true; + i = -1; + while ((i = bms_next_member(relids, i)) >= 0) + { + int top_parent_relid = (int) top_parent_relid_array[i]; + + if (top_parent_relid == 0) + top_parent_relid = i; /* 'i' has no parents, so add itself */ + else if (top_parent_relid != i) + is_top_parent = false; + result = bms_add_member(result, top_parent_relid); + } + + if (is_top_parent) + { + bms_free(result); + return NULL; + } + + return result; +} + + /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 07e2415398..a1f044d8a6 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -248,6 +248,14 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * top_parent_relid_array is the same length as simple_rel_array and holds + * the top-level parent indexes of the corresponding rels within + * simple_rel_array. The element can be zero if the rel has no parents, + * i.e., is itself in a top-level. + */ + Index *top_parent_relid_array pg_node_attr(read_write_ignore); + /* * all_baserels is a Relids set of all base relids (but not joins or * "other" rels) in the query. This is computed in deconstruct_jointree. @@ -957,6 +965,17 @@ typedef struct RelOptInfo /* Bitmask of optional features supported by the table AM */ uint32 amflags; + /* + * information about a join rel + */ + /* index in root->join_rel_list of this rel */ + Index join_rel_list_index; + + /* + * EquivalenceMembers referencing this rel + */ + List *eclass_child_members; + /* * Information about foreign tables and foreign joins */ @@ -1371,6 +1390,12 @@ typedef struct JoinDomain * entry: consider SELECT random() AS a, random() AS b ... ORDER BY b,a. * So we record the SortGroupRef of the originating sort clause. * + * EquivalenceClass->ec_members can only have parent members, and child members + * are stored in RelOptInfos, from which those child members are translated. To + * lookup child EquivalenceMembers, we use EquivalenceChildMemberIterator. See + * its comment for usage. The approach to lookup child members quickly is + * described as iterate_child_rel_equivalences() comment. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1443,10 +1468,46 @@ typedef struct EquivalenceMember bool em_is_child; /* derived version for a child relation? */ Oid em_datatype; /* the "nominal type" used by the opfamily */ JoinDomain *em_jdomain; /* join domain containing the source clause */ + Relids em_child_relids; /* all relids of child rels */ + Bitmapset *em_child_joinrel_relids; /* indexes in root->join_rel_list + * of join rel children */ /* if em_is_child is true, this links to corresponding EM for top parent */ struct EquivalenceMember *em_parent pg_node_attr(read_write_ignore); } EquivalenceMember; +/* + * EquivalenceChildMemberIterator + * + * EquivalenceClass has only parent EquivalenceMembers, so we need to translate + * child members if necessary. EquivalenceChildMemberIterator provides a way to + * iterate over translated child members during the loop in addition to all of + * the parent EquivalenceMembers. + * + * The most common way to use this struct is as follows: + * ----- + * EquivalenceClass *ec = given; + * Relids rel = given; + * EquivalenceChildMemberIterator it; + * EquivalenceMember *em; + * + * setup_eclass_child_member_iterator(&it, ec); + * while ((em = eclass_child_member_iterator_next(&it)) != NULL) + * { + * use em ...; + * if (we need to iterate over child EquivalenceMembers) + * iterate_child_rel_equivalences(&it, root, ec, em, rel); + * use em ...; + * } + * dispose_eclass_child_member_iterator(&it); + * ----- + */ +typedef struct +{ + int index; /* current index within 'ec_members'. */ + bool modified; /* is 'ec_members' a newly allocated one? */ + List *ec_members; /* parent and child members */ +} EquivalenceChildMemberIterator; + /* * PathKeys * diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 1035e6560c..5e79cf1f63 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -332,6 +332,24 @@ extern Relids min_join_parameterization(PlannerInfo *root, extern RelOptInfo *fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids); extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); + +/* + * find_relids_top_parents + * Compute the set of top-parent relids of rel. + * + * Replaces all Relids appearing in the given 'relids' as their top-level + * parents. The result will be NULL if and only if all of the given relids are + * top-level. + * + * The motivation for having this feature as a macro rather than a function is + * that Relids are top-level in most cases. We can quickly determine when + * root->top_parent_relid_array is NULL. + */ +#define find_relids_top_parents(root, relids) \ + (likely((root)->top_parent_relid_array == NULL) \ + ? NULL : find_relids_top_parents_slow(root, relids)) +extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); + extern ParamPathInfo *get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, Relids required_outer); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 54869d4401..50812e3a5d 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -137,7 +137,8 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Index sortref, Relids rel, bool create_it); -extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec, +extern EquivalenceMember *find_ec_member_matching_expr(PlannerInfo *root, + EquivalenceClass *ec, Expr *expr, Relids relids); extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root, @@ -179,6 +180,15 @@ extern void add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, List *child_tlist, List *setop_pathkeys); +extern void setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec); +extern EquivalenceMember *eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it); +extern void dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it); +extern void iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids); extern List *generate_implied_equalities_for_column(PlannerInfo *root, RelOptInfo *rel, ec_matches_callback_type callback, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 57de1acff3..9a0bcecb9a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -683,6 +683,7 @@ EphemeralNamedRelation EphemeralNamedRelationData EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData +EquivalenceChildMemberIterator EquivalenceClass EquivalenceMember ErrorContextCallback -- 2.45.2.windows.1 [application/octet-stream] v27-0002-Introduce-indexes-for-RestrictInfo.patch (42.9K, ../../CAJ2pMkbOoQw0_Kih7Bn3ENgHjLqNmAO-DqG2NUaRyhAEfTMy7Q@mail.gmail.com/3-v27-0002-Introduce-indexes-for-RestrictInfo.patch) download | inline diff: From dbf44c033769fdfc3ebcfbc4930af8b1c98b9540 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:44:25 +0900 Subject: [PATCH v27 2/5] Introduce indexes for RestrictInfo This commit adds indexes to speed up searches for RestrictInfos. When there are many child partitions and we have to perform planning for join operations on the tables, we have to handle a large number of RestrictInfos, resulting in a long planning time. This commit adds ec_[source|derive]_indexes to speed up the search. We can use the indexes to filter out unwanted RestrictInfos, and improve the planning performance. Author: David Rowley <[email protected]> and me, and includes rebase by Alena Rybakina <[email protected]> [1]. [1] https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/nodes/outfuncs.c | 6 +- src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/path/equivclass.c | 516 ++++++++++++++++++++-- src/backend/optimizer/plan/analyzejoins.c | 49 +- src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/appendinfo.c | 2 + src/backend/optimizer/util/inherit.c | 7 + src/backend/optimizer/util/restrictinfo.c | 5 + src/include/nodes/parsenodes.h | 6 + src/include/nodes/pathnodes.h | 41 +- src/include/optimizer/paths.h | 21 +- 13 files changed, 593 insertions(+), 69 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 9827cf16be..bd6a79b57a 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -466,8 +466,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_opfamilies); WRITE_OID_FIELD(ec_collation); WRITE_NODE_FIELD(ec_members); - WRITE_NODE_FIELD(ec_sources); - WRITE_NODE_FIELD(ec_derives); + WRITE_BITMAPSET_FIELD(ec_source_indexes); + WRITE_BITMAPSET_FIELD(ec_derive_indexes); WRITE_BITMAPSET_FIELD(ec_relids); WRITE_BOOL_FIELD(ec_has_const); WRITE_BOOL_FIELD(ec_has_volatile); @@ -573,6 +573,8 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); + WRITE_BITMAPSET_FIELD(eclass_source_indexes); + WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index be5f19dd7f..70864fc1b2 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -434,6 +434,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); + READ_BITMAPSET_FIELD(eclass_source_indexes); + READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 2bb6db1df7..d18f3979b0 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5840,7 +5840,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, if (ec && ec->ec_has_const) { EquivalenceMember *em = fkinfo->fk_eclass_member[i]; - RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + RestrictInfo *rinfo = find_derived_clause_for_ec_member(root, + ec, em); if (rinfo) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index bcd56da3d2..2e84663606 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -42,6 +42,10 @@ static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, Oid datatype); +static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); +static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); static void generate_base_implied_equalities_no_const(PlannerInfo *root, @@ -319,7 +323,6 @@ process_equivalence(PlannerInfo *root, /* If case 1, nothing to do, except add to sources */ if (ec1 == ec2) { - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -330,6 +333,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); return true; } @@ -350,8 +355,10 @@ process_equivalence(PlannerInfo *root, * be found. */ ec1->ec_members = list_concat(ec1->ec_members, ec2->ec_members); - ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); - ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); + ec1->ec_source_indexes = bms_join(ec1->ec_source_indexes, + ec2->ec_source_indexes); + ec1->ec_derive_indexes = bms_join(ec1->ec_derive_indexes, + ec2->ec_derive_indexes); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); ec1->ec_has_const |= ec2->ec_has_const; /* can't need to set has_volatile */ @@ -363,10 +370,9 @@ process_equivalence(PlannerInfo *root, root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; - ec2->ec_sources = NIL; - ec2->ec_derives = NIL; + ec2->ec_source_indexes = NULL; + ec2->ec_derive_indexes = NULL; ec2->ec_relids = NULL; - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -377,13 +383,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec1) { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, jdomain, item2_type); - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -394,13 +401,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec2) { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, jdomain, item1_type); - ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -411,6 +419,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec2, restrictinfo); } else { @@ -420,8 +430,8 @@ process_equivalence(PlannerInfo *root, ec->ec_opfamilies = opfamilies; ec->ec_collation = collation; ec->ec_members = NIL; - ec->ec_sources = list_make1(restrictinfo); - ec->ec_derives = NIL; + ec->ec_source_indexes = NULL; + ec->ec_derive_indexes = NULL; ec->ec_relids = NULL; ec->ec_has_const = false; ec->ec_has_volatile = false; @@ -443,6 +453,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec, restrictinfo); } return true; @@ -584,6 +596,167 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, return em; } +/* + * add_eq_source - add 'rinfo' in eq_sources for this 'ec' + */ +static void +add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int source_idx = list_length(root->eq_sources); + int i; + + ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); + root->eq_sources = lappend(root->eq_sources, rinfo); + Assert(rinfo->eq_sources_index == -1); + rinfo->eq_sources_index = source_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + source_idx); + } +} + +/* + * add_eq_derive - add 'rinfo' in eq_derives for this 'ec' + */ +static void +add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int derive_idx = list_length(root->eq_derives); + int i; + + ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); + root->eq_derives = lappend(root->eq_derives, rinfo); + Assert(rinfo->eq_derives_index == -1); + rinfo->eq_derives_index = derive_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + derive_idx); + } +} + +/* + * update_clause_relids + * Update RestrictInfo->clause_relids with adjusting the relevant indexes. + * The clause_relids field must be updated through this function, not by + * setting the field directly. + */ +void +update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids) +{ + int i; + Relids removing_relids; + Relids adding_relids; +#ifdef USE_ASSERT_CHECKING + Relids common_relids; +#endif + + /* + * If there is nothing to do, we return immediately after setting the + * clause_relids field. + */ + if (rinfo->eq_sources_index == -1 && rinfo->eq_derives_index == -1) + { + rinfo->clause_relids = new_clause_relids; + return; + } + + /* + * Remove any references between this RestrictInfo and RangeTblEntries + * that are no longer relevant. + */ + removing_relids = bms_difference(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(removing_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(removing_relids); + + /* + * Add references between this RestrictInfo and RangeTblEntries that will + * be relevant. + */ + adding_relids = bms_difference(new_clause_relids, rinfo->clause_relids); + i = -1; + while ((i = bms_next_member(adding_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(!bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_add_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(!bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_add_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(adding_relids); + +#ifdef USE_ASSERT_CHECKING + + /* + * Verify that all indexes are set for common Relids. + */ + common_relids = bms_intersect(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(common_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + if (rinfo->eq_derives_index != -1) + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + } + bms_free(common_relids); +#endif + + /* + * Since we have done everything to adjust the indexes, we can set the + * clause_relids field. + */ + rinfo->clause_relids = new_clause_relids; +} + /* * get_eclass_for_sort_expr @@ -729,8 +902,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = collation; newec->ec_members = NIL; - newec->ec_sources = NIL; - newec->ec_derives = NIL; + newec->ec_source_indexes = NULL; + newec->ec_derive_indexes = NULL; newec->ec_relids = NULL; newec->ec_has_const = false; newec->ec_has_volatile = contain_volatile_functions((Node *) expr); @@ -1108,7 +1281,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources or ec_derives for matches. It doesn't really + * don't search eq_sources or eq_derives for matches. It doesn't really * seem worth the trouble to do so. */ void @@ -1201,6 +1374,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *const_em = NULL; ListCell *lc; + int i; /* * In the trivial case where we just had one "var = const" clause, push @@ -1210,9 +1384,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, * equivalent to the old one. */ if (list_length(ec->ec_members) == 2 && - list_length(ec->ec_sources) == 1) + bms_get_singleton_member(ec->ec_source_indexes, &i)) { - RestrictInfo *restrictinfo = (RestrictInfo *) linitial(ec->ec_sources); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, root->eq_sources, i); distribute_restrictinfo_to_rels(root, restrictinfo); return; @@ -1270,9 +1444,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the correct - * markings for a mergejoinable clause, and save it in ec_derives. (We + * markings for a mergejoinable clause, and save it in eq_derives. (We * will not re-use such clauses directly, but selectivity estimation - * may consult the list later. Note that this use of ec_derives does + * may consult the list later. Note that this use of eq_derives does * not overlap with its use for join clauses, since we never generate * join clauses from an ec_has_const eclass.) */ @@ -1282,7 +1456,8 @@ generate_base_implied_equalities_const(PlannerInfo *root, rinfo->left_ec = rinfo->right_ec = ec; rinfo->left_em = cur_em; rinfo->right_em = const_em; - ec->ec_derives = lappend(ec->ec_derives, rinfo); + + add_eq_derive(root, ec, rinfo); } } } @@ -1348,7 +1523,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the * correct markings for a mergejoinable clause. We don't put it - * in ec_derives however; we don't currently need to re-find such + * in eq_derives however; we don't currently need to re-find such * clauses, and we don't want to clutter that list with non-join * clauses. */ @@ -1405,11 +1580,12 @@ static void generate_base_implied_equalities_broken(PlannerInfo *root, EquivalenceClass *ec) { - ListCell *lc; + int i = -1; - foreach(lc, ec->ec_sources) + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); if (ec->ec_has_const || bms_membership(restrictinfo->required_relids) != BMS_MULTIPLE) @@ -1450,11 +1626,11 @@ generate_base_implied_equalities_broken(PlannerInfo *root, * Because the same join clauses are likely to be needed multiple times as * we consider different join paths, we avoid generating multiple copies: * whenever we select a particular pair of EquivalenceMembers to join, - * we check to see if the pair matches any original clause (in ec_sources) - * or previously-built clause (in ec_derives). This saves memory and allows - * re-use of information cached in RestrictInfos. We also avoid generating - * commutative duplicates, i.e. if the algorithm selects "a.x = b.y" but - * we already have "b.y = a.x", we return the existing clause. + * we check to see if the pair matches any original clause in root's + * eq_sources or previously-built clause (in root's eq_derives). This saves + * memory and allows re-use of information cached in RestrictInfos. We also + * avoid generating commutative duplicates, i.e. if the algorithm selects + * "a.x = b.y" but we already have "b.y = a.x", we return the existing clause. * * If we are considering an outer join, sjinfo is the associated OJ info, * otherwise it can be NULL. @@ -1832,12 +2008,16 @@ generate_join_implied_equalities_broken(PlannerInfo *root, Relids nominal_inner_relids, RelOptInfo *inner_rel) { + Bitmapset *matching_es; List *result = NIL; - ListCell *lc; + int i; - foreach(lc, ec->ec_sources) + matching_es = get_ec_source_indexes(root, ec, nominal_join_relids); + i = -1; + while ((i = bms_next_member(matching_es, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); Relids clause_relids = restrictinfo->required_relids; if (bms_is_subset(clause_relids, nominal_join_relids) && @@ -1849,12 +2029,12 @@ generate_join_implied_equalities_broken(PlannerInfo *root, /* * If we have to translate, just brute-force apply adjust_appendrel_attrs * to all the RestrictInfos at once. This will result in returning - * RestrictInfos that are not listed in ec_derives, but there shouldn't be + * RestrictInfos that are not listed in eq_derives, but there shouldn't be * any duplication, and it's a sufficiently narrow corner case that we * shouldn't sweat too much over it anyway. * * Since inner_rel might be an indirect descendant of the baserel - * mentioned in the ec_sources clauses, we have to be prepared to apply + * mentioned in the eq_sources clauses, we have to be prepared to apply * multiple levels of Var translation. */ if (IS_OTHER_REL(inner_rel) && result != NIL) @@ -1916,10 +2096,11 @@ create_join_clause(PlannerInfo *root, EquivalenceMember *rightem, EquivalenceClass *parent_ec) { + Bitmapset *matches; RestrictInfo *rinfo; RestrictInfo *parent_rinfo = NULL; - ListCell *lc; MemoryContext oldcontext; + int i; /* * Search to see if we already built a RestrictInfo for this pair of @@ -1930,9 +2111,12 @@ create_join_clause(PlannerInfo *root, * it's not identical, it'd better have the same effects, or the operator * families we're using are broken. */ - foreach(lc, ec->ec_sources) + matches = bms_int_members(get_ec_source_indexes_strict(root, ec, leftem->em_relids), + get_ec_source_indexes_strict(root, ec, rightem->em_relids)); + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -1943,9 +2127,13 @@ create_join_clause(PlannerInfo *root, return rinfo; } - foreach(lc, ec->ec_derives) + matches = bms_int_members(get_ec_derive_indexes_strict(root, ec, leftem->em_relids), + get_ec_derive_indexes_strict(root, ec, rightem->em_relids)); + + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -2018,7 +2206,7 @@ create_join_clause(PlannerInfo *root, rinfo->left_em = leftem; rinfo->right_em = rightem; /* and save it for possible re-use */ - ec->ec_derives = lappend(ec->ec_derives, rinfo); + add_eq_derive(root, ec, rinfo); MemoryContextSwitchTo(oldcontext); @@ -2744,16 +2932,19 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, * Returns NULL if no such clause can be found. */ RestrictInfo * -find_derived_clause_for_ec_member(EquivalenceClass *ec, +find_derived_clause_for_ec_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em) { - ListCell *lc; + int i; Assert(ec->ec_has_const); Assert(!em->em_is_const); - foreach(lc, ec->ec_derives) + + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); /* * generate_base_implied_equalities_const will have put non-const @@ -3467,7 +3658,7 @@ have_relevant_eclass_joinclause(PlannerInfo *root, * surely be true if both of them overlap ec_relids.) * * Note we don't test ec_broken; if we did, we'd need a separate code - * path to look through ec_sources. Checking the membership anyway is + * path to look through eq_sources. Checking the membership anyway is * OK as a possibly-overoptimistic heuristic. * * We don't test ec_has_const either, even though a const eclass won't @@ -3555,7 +3746,7 @@ eclass_useful_for_merging(PlannerInfo *root, /* * Note we don't test ec_broken; if we did, we'd need a separate code path - * to look through ec_sources. Checking the members anyway is OK as a + * to look through eq_sources. Checking the members anyway is OK as a * possibly-overoptimistic heuristic. */ @@ -3712,3 +3903,240 @@ get_common_eclass_indexes(PlannerInfo *root, Relids relids1, Relids relids2) /* Calculate and return the common EC indexes, recycling the left input. */ return bms_int_members(rel1ecs, rel2ecs); } + +/* + * get_ec_source_indexes + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_esis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_esis, ec->ec_source_indexes); +} + +/* + * get_ec_source_indexes_strict + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *esis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + esis = bms_intersect(ec->ec_source_indexes, + rte->eclass_source_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + esis = bms_int_members(esis, rte->eclass_source_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return esis; +} + +/* + * get_ec_derive_indexes + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + * + * XXX is this function even needed? + */ +Bitmapset * +get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_edis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_edis, ec->ec_derive_indexes); +} + +/* + * get_ec_derive_indexes_strict + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *edis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + edis = bms_intersect(ec->ec_derive_indexes, + rte->eclass_derive_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + edis = bms_int_members(edis, rte->eclass_derive_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return edis; +} + +#ifdef USE_ASSERT_CHECKING +/* + * verify_eclass_indexes + * Verify that there are no missing references between RestrictInfos and + * EquivalenceMember's indexes, namely eclass_source_indexes and + * eclass_derive_indexes. If you modify these indexes, you should check + * them with this function. + */ +void +verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) +{ + ListCell *lc; + + /* + * All RestrictInfos in root->eq_sources must have references to + * eclass_source_indexes. + */ + foreach(lc, root->eq_sources) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_sources, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + } + } + + /* + * All RestrictInfos in root->eq_derives must have references to + * eclass_derive_indexes. + */ + foreach(lc, root->eq_derives) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_derives, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + } + } +} +#endif diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 928d926645..c8f7435909 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -36,9 +36,10 @@ static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); static void remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); -static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, +static void remove_rel_from_restrictinfo(PlannerInfo *root, + RestrictInfo *rinfo, int relid, int ojrelid); -static void remove_rel_from_eclass(EquivalenceClass *ec, +static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static bool rel_supports_distinctness(PlannerInfo *root, RelOptInfo *rel); @@ -476,7 +477,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * that any such PHV is safe (and updated its ph_eval_at), so we * can just drop those references. */ - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); /* * Cross-check that the clause itself does not reference the @@ -505,7 +506,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) if (bms_is_member(relid, ec->ec_relids) || bms_is_member(ojrelid, ec->ec_relids)) - remove_rel_from_eclass(ec, relid, ojrelid); + remove_rel_from_eclass(root, ec, relid, ojrelid); } /* @@ -578,19 +579,28 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * we have to also clean up the sub-clauses. */ static void -remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) +remove_rel_from_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo, int relid, + int ojrelid) { + Relids new_clause_relids; + /* * initsplan.c is fairly cavalier about allowing RestrictInfos to share * relid sets with other RestrictInfos, and SpecialJoinInfos too. Make * sure this RestrictInfo has its own relid sets before we modify them. - * (In present usage, clause_relids is probably not shared, but - * required_relids could be; let's not assume anything.) + * The 'new_clause_relids' passed to update_clause_relids() must be a + * different instance from the current rinfo->clause_relids, so we make a + * copy of it. + */ + new_clause_relids = bms_copy(rinfo->clause_relids); + new_clause_relids = bms_del_member(new_clause_relids, relid); + new_clause_relids = bms_del_member(new_clause_relids, ojrelid); + update_clause_relids(root, rinfo, new_clause_relids); + + /* + * In present usage, required_relids could be shared, so we make a copy of + * it. */ - rinfo->clause_relids = bms_copy(rinfo->clause_relids); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, relid); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, ojrelid); - /* Likewise for required_relids */ rinfo->required_relids = bms_copy(rinfo->required_relids); rinfo->required_relids = bms_del_member(rinfo->required_relids, relid); rinfo->required_relids = bms_del_member(rinfo->required_relids, ojrelid); @@ -615,14 +625,14 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) { RestrictInfo *rinfo2 = lfirst_node(RestrictInfo, lc2); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } else { RestrictInfo *rinfo2 = castNode(RestrictInfo, orarg); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } } @@ -638,9 +648,11 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) * level(s). */ static void -remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) +remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, + int ojrelid) { ListCell *lc; + int i; /* Fix up the EC's overall relids */ ec->ec_relids = bms_del_member(ec->ec_relids, relid); @@ -667,11 +679,12 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) } /* Fix up the source clauses, in case we can re-use them later */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); } /* @@ -679,7 +692,7 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) * drop them. (At this point, any such clauses would be base restriction * clauses, which we'd not need anymore anyway.) */ - ec->ec_derives = NIL; + ec->ec_derive_indexes = NULL; } /* diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 0f423e9684..51c4a4145d 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -659,6 +659,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, PlannerInfo *parent_root, root->multiexpr_params = NIL; root->join_domains = NIL; root->eq_classes = NIL; + root->eq_sources = NIL; + root->eq_derives = NIL; root->ec_merging_done = false; root->last_rinfo_serial = 0; root->all_result_relids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 4d7f972caf..f17e9e2ab7 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1137,6 +1137,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, subroot->multiexpr_params = NIL; subroot->join_domains = NIL; subroot->eq_classes = NIL; + subroot->eq_sources = NIL; + subroot->eq_derives = NIL; subroot->ec_merging_done = false; subroot->last_rinfo_serial = 0; subroot->all_result_relids = NULL; diff --git a/src/backend/optimizer/util/appendinfo.c b/src/backend/optimizer/util/appendinfo.c index 4989722637..4489c2e1bf 100644 --- a/src/backend/optimizer/util/appendinfo.c +++ b/src/backend/optimizer/util/appendinfo.c @@ -495,6 +495,8 @@ adjust_appendrel_attrs_mutator(Node *node, newinfo->right_bucketsize = -1; newinfo->left_mcvfreq = -1; newinfo->right_mcvfreq = -1; + newinfo->eq_sources_index = -1; + newinfo->eq_derives_index = -1; return (Node *) newinfo; } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5b..b5d3984219 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,6 +492,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); + /* + * We do not want to inherit the EquivalenceMember indexes of the parent + * to its child + */ + childrte->eclass_source_indexes = NULL; + childrte->eclass_derive_indexes = NULL; + Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 0b406e9334..59ba503ecb 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -247,6 +247,9 @@ make_restrictinfo_internal(PlannerInfo *root, restrictinfo->left_hasheqoperator = InvalidOid; restrictinfo->right_hasheqoperator = InvalidOid; + restrictinfo->eq_sources_index = -1; + restrictinfo->eq_derives_index = -1; + return restrictinfo; } @@ -403,6 +406,8 @@ commute_restrictinfo(RestrictInfo *rinfo, Oid comm_op) result->right_mcvfreq = rinfo->left_mcvfreq; result->left_hasheqoperator = InvalidOid; result->right_hasheqoperator = InvalidOid; + result->eq_sources_index = -1; + result->eq_derives_index = -1; return result; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index c92cef3d16..504308a84b 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1247,6 +1247,12 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); + Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources + * list for RestrictInfos that mention + * this relation */ + Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives + * list for RestrictInfos that mention + * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index a1f044d8a6..0d06bb150b 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -321,6 +321,12 @@ struct PlannerInfo /* list of active EquivalenceClasses */ List *eq_classes; + /* list of source RestrictInfos used to build EquivalenceClasses */ + List *eq_sources; + + /* list of RestrictInfos derived from EquivalenceClasses */ + List *eq_derives; + /* set true once ECs are canonical */ bool ec_merging_done; @@ -1396,6 +1402,24 @@ typedef struct JoinDomain * its comment for usage. The approach to lookup child members quickly is * described as iterate_child_rel_equivalences() comment. * + * At various locations in the query planner, we must search for source and + * derived RestrictInfos regarding a given EquivalenceClass. For the common + * case, an EquivalenceClass does not have a large number of RestrictInfos, + * however, in cases such as planning queries to partitioned tables, the + * number of members can become large. To maintain planning performance, we + * make use of a bitmap index to allow us to quickly find RestrictInfos in a + * given EquivalenceClass belonging to a given relation or set of relations. + * This is done by storing a list of RestrictInfos belonging to all + * EquivalenceClasses in PlannerInfo and storing a Bitmapset for each + * RelOptInfo which has a bit set for each RestrictInfo in that list which + * relates to the given relation. We also store a Bitmapset to mark all of + * the indexes in the PlannerInfo's list of RestrictInfos in the + * EquivalenceClass. We can quickly find the interesting indexes into the + * PlannerInfo's list by performing a bit-wise AND on the RelOptInfo's + * Bitmapset and the EquivalenceClasses. RestrictInfos must be looked up in + * PlannerInfo by this technique using the ec_source_indexes and + * ec_derive_indexes Bitmapsets. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1414,8 +1438,10 @@ typedef struct EquivalenceClass List *ec_opfamilies; /* btree operator family OIDs */ Oid ec_collation; /* collation, if datatypes are collatable */ List *ec_members; /* list of EquivalenceMembers */ - List *ec_sources; /* list of generating RestrictInfos */ - List *ec_derives; /* list of derived RestrictInfos */ + Bitmapset *ec_source_indexes; /* indexes into PlannerInfo's eq_sources + * list of generating RestrictInfos */ + Bitmapset *ec_derive_indexes; /* indexes into PlannerInfo's eq_derives + * list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except * for child members (see below) */ bool ec_has_const; /* any pseudoconstants in ec_members? */ @@ -2656,7 +2682,12 @@ typedef struct RestrictInfo /* number of base rels in clause_relids */ int num_base_rels pg_node_attr(equal_ignore); - /* The relids (varnos+varnullingrels) actually referenced in the clause: */ + /* + * The relids (varnos+varnullingrels) actually referenced in the clause. + * + * NOTE: This field must be updated through update_clause_relids(), not by + * setting the field directly. + */ Relids clause_relids pg_node_attr(equal_ignore); /* The set of relids required to evaluate the clause: */ @@ -2774,6 +2805,10 @@ typedef struct RestrictInfo /* hash equality operators used for memoize nodes, else InvalidOid */ Oid left_hasheqoperator pg_node_attr(equal_ignore); Oid right_hasheqoperator pg_node_attr(equal_ignore); + + /* the index within root->eq_sources and root->eq_derives */ + int eq_sources_index pg_node_attr(equal_ignore); + int eq_derives_index pg_node_attr(equal_ignore); } RestrictInfo; /* diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 50812e3a5d..a24a5801c5 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -129,6 +129,8 @@ extern Expr *canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation); extern void reconsider_outer_join_clauses(PlannerInfo *root); extern void rebuild_eclass_attr_needed(PlannerInfo *root); +extern void update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids); extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Expr *expr, List *opfamilies, @@ -165,7 +167,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); -extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, +extern RestrictInfo *find_derived_clause_for_ec_member(PlannerInfo *root, + EquivalenceClass *ec, EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, @@ -204,6 +207,22 @@ extern bool eclass_useful_for_merging(PlannerInfo *root, extern bool is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist); extern bool is_redundant_with_indexclauses(RestrictInfo *rinfo, List *indexclauses); +extern Bitmapset *get_ec_source_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_source_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +#ifdef USE_ASSERT_CHECKING +extern void verify_eclass_indexes(PlannerInfo *root, + EquivalenceClass *ec); +#endif /* * pathkeys.c -- 2.45.2.windows.1 [application/octet-stream] v27-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch (14.4K, ../../CAJ2pMkbOoQw0_Kih7Bn3ENgHjLqNmAO-DqG2NUaRyhAEfTMy7Q@mail.gmail.com/4-v27-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch) download | inline diff: From 66f4d2f89119184ee1968a67c1fe4464ba8c498f Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 19 Jan 2024 11:43:29 +0900 Subject: [PATCH v27 3/5] Move EquivalenceClass indexes to PlannerInfo In the previous commit, the indexes, namely ec_[source|derive]_indexes, were in RestrictInfo. This was a workaround because RelOptInfo was the best place but some RelOptInfos can be NULL and we cannot store indexes for them. This commit introduces a new struct, EquivalenceClassIndexes. This struct exists in PlannerInfo and holds our indexes. This change prevents a serialization problem with RestirctInfo in the previous commit. --- src/backend/nodes/outfuncs.c | 2 - src/backend/nodes/readfuncs.c | 2 - src/backend/optimizer/path/equivclass.c | 83 ++++++++++++------------- src/backend/optimizer/util/inherit.c | 7 --- src/backend/optimizer/util/relnode.c | 6 ++ src/include/nodes/parsenodes.h | 6 -- src/include/nodes/pathnodes.h | 26 ++++++++ src/tools/pgindent/typedefs.list | 1 + 8 files changed, 74 insertions(+), 59 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index bd6a79b57a..8a635a2a12 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -573,8 +573,6 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); - WRITE_BITMAPSET_FIELD(eclass_source_indexes); - WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 70864fc1b2..be5f19dd7f 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -434,8 +434,6 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_BITMAPSET_FIELD(eclass_source_indexes); - READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 2e84663606..679cf34174 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -613,10 +613,10 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, - source_idx); + index->source_indexes = bms_add_member(index->source_indexes, + source_idx); } } @@ -637,10 +637,10 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, - derive_idx); + index->derive_indexes = bms_add_member(index->derive_indexes, + derive_idx); } } @@ -679,22 +679,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(removing_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_del_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_del_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_del_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_del_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -708,22 +708,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(adding_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(!bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_add_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_add_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(!bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_add_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_add_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -738,14 +738,14 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(common_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); + index->source_indexes)); if (rinfo->eq_derives_index != -1) Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); + index->derive_indexes)); } bms_free(common_relids); #endif @@ -3920,9 +3920,9 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + rel_esis = bms_add_members(rel_esis, index->source_indexes); } #ifdef USE_ASSERT_CHECKING @@ -3958,7 +3958,7 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -3967,12 +3967,12 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ esis = bms_intersect(ec->ec_source_indexes, - rte->eclass_source_indexes); + index->source_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - esis = bms_int_members(esis, rte->eclass_source_indexes); + index = &root->eclass_indexes_array[i]; + esis = bms_int_members(esis, index->source_indexes); } } @@ -4009,9 +4009,9 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + rel_edis = bms_add_members(rel_edis, index->derive_indexes); } #ifdef USE_ASSERT_CHECKING @@ -4047,7 +4047,7 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -4056,12 +4056,12 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, - rte->eclass_derive_indexes); + index->derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - edis = bms_int_members(edis, rte->eclass_derive_indexes); + index = &root->eclass_indexes_array[i]; + edis = bms_int_members(edis, index->derive_indexes); } } @@ -4084,9 +4084,8 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, /* * verify_eclass_indexes * Verify that there are no missing references between RestrictInfos and - * EquivalenceMember's indexes, namely eclass_source_indexes and - * eclass_derive_indexes. If you modify these indexes, you should check - * them with this function. + * EquivalenceMember's indexes, namely source_indexes and derive_indexes. + * If you modify these indexes, you should check them with this function. */ void verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) @@ -4095,7 +4094,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) /* * All RestrictInfos in root->eq_sources must have references to - * eclass_source_indexes. + * source_indexes. */ foreach(lc, root->eq_sources) { @@ -4112,13 +4111,13 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].source_indexes)); } } /* * All RestrictInfos in root->eq_derives must have references to - * eclass_derive_indexes. + * derive_indexes. */ foreach(lc, root->eq_derives) { @@ -4135,7 +4134,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].derive_indexes)); } } } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index b5d3984219..3c2198ea5b 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,13 +492,6 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); - /* - * We do not want to inherit the EquivalenceMember indexes of the parent - * to its child - */ - childrte->eclass_source_indexes = NULL; - childrte->eclass_derive_indexes = NULL; - Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index f32575b949..c219de44d7 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -119,6 +119,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->simple_rte_array[rti++] = rte; } + root->eclass_indexes_array = (EquivalenceClassIndexes *) + palloc0(size * sizeof(EquivalenceClassIndexes)); + /* append_rel_array is not needed if there are no AppendRelInfos */ if (root->append_rel_list == NIL) { @@ -218,6 +221,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->top_parent_relid_array = NULL; } + root->eclass_indexes_array = + repalloc0_array(root->eclass_indexes_array, EquivalenceClassIndexes, root->simple_rel_array_size, new_size); + root->simple_rel_array_size = new_size; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 504308a84b..c92cef3d16 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1247,12 +1247,6 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); - Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources - * list for RestrictInfos that mention - * this relation */ - Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives - * list for RestrictInfos that mention - * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 0d06bb150b..3eeaa7068b 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -192,6 +192,8 @@ typedef struct PlannerInfo PlannerInfo; #define HAVE_PLANNERINFO_TYPEDEF 1 #endif +struct EquivalenceClassIndexes; + struct PlannerInfo { pg_node_attr(no_copy_equal, no_read, no_query_jumble) @@ -248,6 +250,13 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * eclass_indexes_array is the same length as simple_rel_array and holds + * the indexes of the corresponding rels for faster lookups of + * RestrictInfo. See the EquivalenceClass comment for more details. + */ + struct EquivalenceClassIndexes *eclass_indexes_array pg_node_attr(read_write_ignore); + /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within @@ -1534,6 +1543,23 @@ typedef struct List *ec_members; /* parent and child members */ } EquivalenceChildMemberIterator; +/* + * EquivalenceClassIndexes + * + * As mentioned in the EquivalenceClass comment, we introduce a + * bitmapset-based indexing mechanism for faster lookups of RestrictInfo. This + * struct exists for each relation and holds the corresponding indexes. + */ +typedef struct EquivalenceClassIndexes +{ + Bitmapset *source_indexes; /* Indexes in PlannerInfo's eq_sources list + * for RestrictInfos that mention this + * relation */ + Bitmapset *derive_indexes; /* Indexes in PlannerInfo's eq_derives list + * for RestrictInfos that mention this + * relation */ +} EquivalenceClassIndexes; + /* * PathKeys * diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 9a0bcecb9a..3e92b5a27a 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -685,6 +685,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceChildMemberIterator EquivalenceClass +EquivalenceClassIndexes EquivalenceMember ErrorContextCallback ErrorData -- 2.45.2.windows.1 [application/octet-stream] v27-0004-Rename-add_eq_member-to-add_parent_eq_member.patch (5.2K, ../../CAJ2pMkbOoQw0_Kih7Bn3ENgHjLqNmAO-DqG2NUaRyhAEfTMy7Q@mail.gmail.com/5-v27-0004-Rename-add_eq_member-to-add_parent_eq_member.patch) download | inline diff: From 56bafc290624114ae5fa86782b36b7b3a515ef4d Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Wed, 21 Aug 2024 13:54:59 +0900 Subject: [PATCH v27 4/5] Rename add_eq_member() to add_parent_eq_member() After our changes, add_eq_member() no longer creates and adds child EquivalenceMembers. This commit renames it to add_parent_eq_member() to clarify that it only creates parent members, and that we need to use make_eq_member() to handle child EquivalenceMembers. --- src/backend/optimizer/path/equivclass.c | 54 ++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 679cf34174..c54ebd7432 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -38,10 +38,10 @@ static EquivalenceMember *make_eq_member(EquivalenceClass *ec, JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype); -static EquivalenceMember *add_eq_member(EquivalenceClass *ec, - Expr *expr, Relids relids, - JoinDomain *jdomain, - Oid datatype); +static EquivalenceMember *add_parent_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + Oid datatype); static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo); static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, @@ -389,8 +389,8 @@ process_equivalence(PlannerInfo *root, else if (ec1) { /* Case 3: add item2 to ec1 */ - em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, item2_type); + em2 = add_parent_eq_member(ec1, item2, item2_relids, + jdomain, item2_type); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -407,8 +407,8 @@ process_equivalence(PlannerInfo *root, else if (ec2) { /* Case 3: add item1 to ec2 */ - em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, item1_type); + em1 = add_parent_eq_member(ec2, item1, item1_relids, + jdomain, item1_type); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -440,10 +440,10 @@ process_equivalence(PlannerInfo *root, ec->ec_min_security = restrictinfo->security_level; ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; - em1 = add_eq_member(ec, item1, item1_relids, - jdomain, item1_type); - em2 = add_eq_member(ec, item2, item2_relids, - jdomain, item2_type); + em1 = add_parent_eq_member(ec, item1, item1_relids, + jdomain, item1_type); + em2 = add_parent_eq_member(ec, item2, item2_relids, + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -578,7 +578,7 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, } /* - * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * add_parent_eq_member - build a new parent EquivalenceMember and add it to an EC * * Note: We don't have a function to add a child member like * add_child_eq_member() because how to do it depends on the relations they are @@ -586,8 +586,8 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, * add_child_join_rel_equivalences() to see how to add child members. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, Oid datatype) +add_parent_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) { EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, NULL, datatype); @@ -921,14 +921,14 @@ get_eclass_for_sort_expr(PlannerInfo *root, */ expr_relids = pull_varnos(root, (Node *) expr); - newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, opcintype); + newem = add_parent_eq_member(newec, copyObject(expr), expr_relids, + jdomain, opcintype); /* - * add_eq_member doesn't check for volatile functions, set-returning - * functions, aggregates, or window functions, but such could appear in - * sort expressions; so we have to check whether its const-marking was - * correct. + * add_parent_eq_member doesn't check for volatile functions, + * set-returning functions, aggregates, or window functions, but such + * could appear in sort expressions; so we have to check whether its + * const-marking was correct. */ if (newec->ec_has_const) { @@ -3267,12 +3267,12 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + add_parent_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); lc2 = lnext(setop_pathkeys, lc2); } -- 2.45.2.windows.1 [application/octet-stream] v27-0005-Resolve-conflict-with-commit-66c0185.patch (6.4K, ../../CAJ2pMkbOoQw0_Kih7Bn3ENgHjLqNmAO-DqG2NUaRyhAEfTMy7Q@mail.gmail.com/6-v27-0005-Resolve-conflict-with-commit-66c0185.patch) download | inline diff: From 44f04135490cd42ba79e3537183bca4f8e0c44ea Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Tue, 27 Aug 2024 13:20:29 +0900 Subject: [PATCH v27 5/5] Resolve conflict with commit 66c0185 This commit resolves a conflict with 66c0185, which added add_setop_child_rel_equivalences(). The function creates child EquivalenceMembers to efficiently implement UNION queries. This commit adjusts our optimization to handle this type of child EquivalenceMembers based on UNION parent-child relationships. --- src/backend/optimizer/path/equivclass.c | 51 ++++++++++++++++++++++--- src/backend/optimizer/util/inherit.c | 8 +++- src/backend/optimizer/util/relnode.c | 10 +++-- src/include/nodes/pathnodes.h | 2 +- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index c54ebd7432..5ff55d71ee 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -3250,7 +3250,9 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, { TargetEntry *tle = lfirst_node(TargetEntry, lc); EquivalenceMember *parent_em; + EquivalenceMember *child_em; PathKey *pk; + Index parent_relid; if (tle->resjunk) continue; @@ -3267,12 +3269,49 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_parent_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + child_em = make_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); + child_rel->eclass_child_members = + lappend(child_rel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated using + * 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + parent_em->em_child_relids = + bms_add_member(parent_em->em_child_relids, child_rel->relid); + + /* + * Make an UNION parent-child relationship between parent_em and + * child_rel->relid. We record this relationship in + * root->top_parent_relid_array, which generally has AppendRelInfo + * relationships. We use the same array here to retrieve UNION child + * members. + * + * XXX Here we treat the first member of parent_em->em_relids as a + * parent of child_rel. Is this correct? What happens if + * parent_em->em_relids has two or more members? + */ + parent_relid = bms_next_member(parent_em->em_relids, -1); + if (root->top_parent_relid_array == NULL) + { + /* + * If the array is NULL, allocate it here. + */ + root->top_parent_relid_array = (Index *) + palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[child_rel->relid] == -1 || + root->top_parent_relid_array[child_rel->relid] == parent_relid); + root->top_parent_relid_array[child_rel->relid] = parent_relid; lc2 = lnext(setop_pathkeys, lc2); } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5b..7b2390687e 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -582,9 +582,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, * Find a top parent rel's index and save it to top_parent_relid_array. */ if (root->top_parent_relid_array == NULL) + { root->top_parent_relid_array = - (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); - Assert(root->top_parent_relid_array[childRTindex] == 0); + (Index *) palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[childRTindex] == -1); topParentRTindex = parentRTindex; while (root->append_rel_array[topParentRTindex] != NULL && root->append_rel_array[topParentRTindex]->parent_relid != 0) diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index c219de44d7..5ea9a41008 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -133,7 +133,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); root->top_parent_relid_array = (Index *) - palloc0(size * sizeof(Index)); + palloc(size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * size); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -206,7 +208,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); root->top_parent_relid_array = - repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + repalloc_array(root->top_parent_relid_array, Index, new_size); + MemSet(root->top_parent_relid_array + root->simple_rel_array_size, -1, + sizeof(Index) * add_size); } else { @@ -1608,7 +1612,7 @@ find_relids_top_parents_slow(PlannerInfo *root, Relids relids) { int top_parent_relid = (int) top_parent_relid_array[i]; - if (top_parent_relid == 0) + if (top_parent_relid == -1) top_parent_relid = i; /* 'i' has no parents, so add itself */ else if (top_parent_relid != i) is_top_parent = false; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3eeaa7068b..8dc32eb606 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -260,7 +260,7 @@ struct PlannerInfo /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within - * simple_rel_array. The element can be zero if the rel has no parents, + * simple_rel_array. The element can be -1 if the rel has no parents, * i.e., is itself in a top-level. */ Index *top_parent_relid_array pg_node_attr(read_write_ignore); -- 2.45.2.windows.1 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-11-27 19:51 Dmitry Dolgov <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Dmitry Dolgov @ 2024-11-27 19:51 UTC (permalink / raw) To: Yuya Watari <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>; jian he <[email protected]>; Ashutosh Bapat <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> > On Tue, Oct 15, 2024 at 12:20:04PM GMT, Yuya Watari wrote: > > The previous patches do not apply to the current master, so I have > attached the rebased version. Thanks for keeping it up to date. > v25-0001 > This patch is one of the main parts of my optimization. Traditionally, > EquivalenceClass has both parent and child members. However, this > leads to high iteration costs when there are many child partitions. In > v25-0001, EquivalenceClasses no longer have child members. If we need > to iterate over child EquivalenceMembers, we use the > EquivalenceChildMemberIterator and access the children through the > iterator. For more details, see [1] (note that there are some design > changes from [1]). The referenced email containst some benchmark results. But shouldn't the benchmark be repeated after those design changes you're talking about? Few random notes after quickly looking through the first patch: * There are patterns like this scattered around, it looks somewhat confusing: + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); It's not immediately clear which part of get_eclass_for_sort_expr is relevant, or one have to read the whole function first. Probably better to omit the superficial commentary on the call site, and instead expand the commentary for the find_relids_top_parents itself? * The patch series features likely/unlikely since v20, but don't see any discussion about that. Did you notice any visible boost from that? I wonder how necessary that is. ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-12-02 08:51 Yuya Watari <[email protected]> parent: Dmitry Dolgov <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Yuya Watari @ 2024-12-02 08:51 UTC (permalink / raw) To: Dmitry Dolgov <[email protected]>; +Cc: PostgreSQL Developers <[email protected]>; jian he <[email protected]>; Ashutosh Bapat <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> Hello Dmitry, I really appreciate you reviewing these patches. On Thu, Nov 28, 2024 at 4:51 AM Dmitry Dolgov <[email protected]> wrote: > > Few random notes after quickly looking through the first patch: > > * There are patterns like this scattered around, it looks somewhat confusing: > > + /* See the comments in get_eclass_for_sort_expr() to see how this works. */ > + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); > > It's not immediately clear which part of get_eclass_for_sort_expr is > relevant, or one have to read the whole function first. Probably better to > omit the superficial commentary on the call site, and instead expand the > commentary for the find_relids_top_parents itself? Thank you for pointing this out. As you said, those comments are not helpful for understanding, and I also agree that expanding the actual comments is better. I fixed this in v28. > * The patch series features likely/unlikely since v20, but don't see any > discussion about that. Did you notice any visible boost from that? I wonder > how necessary that is. I'm sorry I haven't tested the effects of these likely/unlikely. To discuss the need for them, I have added the removals of these likely/unlikely as v28-0006 and v28-0007, which are attached to this email. I would like to decide whether to adopt these removals after discussion here. There are two uses of likely and unlikely in the original patches: 1. likely in the find_relids_top_parents() macro (quoted below) The find_relids_top_parents() macro has likely. This macro replaces the given Relids as their parents. If they are all already parents, it simply returns NULL. This is to avoid slowing down for non-partitioned cases. For such cases, we don't need to do anything to introduce child EquivalenceMembers. The patches effectively skip this by checking whether the returned Relids is NULL. If there are no partitioned tables in the given query, root->top_parent_relid_array is set to NULL and the macro can return NULL immediately. I assumed that non-partitioned use cases are common, I used likely here. In the attached patches, I removed this in v28-0006 to confirm its effects. ===== #define find_relids_top_parents(root, relids) \ (likely((root)->top_parent_relid_array == NULL) \ ? NULL : find_relids_top_parents_slow(root, relids)) extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); ===== 2. unlikely, which checks if top_parent_relids is null (quoted below) The caller of the find_relids_top_parents() macro has unlikely for the same reason. I removed this in v28-0007. ===== top_parent_rel = find_relids_top_parents(root, rel); ... /* * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && bms_equal(cur_em->em_relids, top_parent_rel)) iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel); ===== > The referenced email containst some benchmark results. But shouldn't the > benchmark be repeated after those design changes you're talking about? Thank you for pointing this out. I ran the same benchmarks as in [1]. 1. Versions used in the benchmarks I used the following five versions in the benchmarks: * Master, * v23: The version just before the design change (introduction of an iterator over child EquivalenceMembers). I rebased this old version for these benchmarks, * ~v28-0005: Same as v27, * ~v28-0006: ~v28-0005 with "likely" removed, * ~v28-0007: ~v28-0006 with "unlikely" removed. 2. Small size cases (make installcheck) This benchmark is to confirm regressions with the patches. I ran 'make installcheck' and measured the planning times of all queries executed during the tests. Table 1: Total planning time for installcheck (seconds) -------------------------------------------- Program | Mean | Median | Stddev -------------------------------------------- Master | 0.920580 | 0.920022 | 0.007805 v23 | 0.922827 | 0.922795 | 0.006311 ~v28-0005 | 0.927891 | 0.927387 | 0.007835 ~v28-0006 | 0.926924 | 0.926663 | 0.007905 ~v28-0007 | 0.928182 | 0.926434 | 0.010324 -------------------------------------------- Table 2: Speedup for installcheck (higher is better) ---------------------------- Program | Mean | Median ---------------------------- v23 | 99.8% | 99.7% ~v28-0005 | 99.2% | 99.2% ~v28-0006 | 99.3% | 99.3% ~v28-0007 | 99.2% | 99.3% ---------------------------- 3. Large size cases (queries A and B) I also evaluated the patches using queries A and B, which can be found at [2]. Both queries join partitioned tables. Table 3: Planning time of query A (n: the number of partitions of each table) (lower is better) ------------------------------------------------------------- n | Master | v23 | ~v28-0005 | ~v28-0006 | ~v28-0007 ------------------------------------------------------------- 1 | 0.245 | 0.250 | 0.253 | 0.252 | 0.250 2 | 0.266 | 0.273 | 0.274 | 0.274 | 0.275 4 | 0.339 | 0.351 | 0.352 | 0.350 | 0.350 8 | 0.435 | 0.443 | 0.446 | 0.442 | 0.443 16 | 0.619 | 0.623 | 0.625 | 0.626 | 0.625 32 | 1.065 | 1.009 | 1.018 | 1.022 | 1.025 64 | 2.520 | 2.197 | 2.196 | 2.202 | 2.204 128 | 6.076 | 4.634 | 4.610 | 4.588 | 4.563 256 | 17.447 | 10.416 | 10.576 | 10.463 | 10.545 384 | 33.068 | 16.023 | 16.019 | 16.015 | 16.092 512 | 57.213 | 21.793 | 21.930 | 21.882 | 21.972 640 | 96.241 | 28.208 | 28.441 | 28.205 | 28.308 768 | 154.046 | 34.724 | 35.059 | 35.012 | 35.001 896 | 241.509 | 49.596 | 49.902 | 49.631 | 49.829 1024 | 319.269 | 48.297 | 48.892 | 48.417 | 48.384 ------------------------------------------------------------- Table 4: Speedup of query A (higher is better) --------------------------------------------------- n | v23 | ~v28-0005 | ~v28-0006 | ~v28-0007 --------------------------------------------------- 1 | 98.1% | 96.6% | 97.2% | 97.8% 2 | 97.3% | 96.9% | 97.0% | 96.8% 4 | 96.8% | 96.3% | 97.0% | 96.9% 8 | 98.0% | 97.4% | 98.4% | 98.2% 16 | 99.3% | 99.0% | 98.8% | 99.0% 32 | 105.6% | 104.7% | 104.2% | 103.9% 64 | 114.7% | 114.8% | 114.4% | 114.4% 128 | 131.1% | 131.8% | 132.4% | 133.2% 256 | 167.5% | 165.0% | 166.8% | 165.5% 384 | 206.4% | 206.4% | 206.5% | 205.5% 512 | 262.5% | 260.9% | 261.5% | 260.4% 640 | 341.2% | 338.4% | 341.2% | 340.0% 768 | 443.6% | 439.4% | 440.0% | 440.1% 896 | 487.0% | 484.0% | 486.6% | 484.7% 1024 | 661.0% | 653.0% | 659.4% | 659.9% --------------------------------------------------- Table 5: Planning time of query B (lower is better) ------------------------------------------------------------- n | Master | v23 | ~v28-0005 | ~v28-0006 | ~v28-0007 ------------------------------------------------------------- 1 | 11.759 | 11.952 | 12.017 | 11.999 | 11.975 2 | 11.334 | 11.474 | 11.538 | 11.525 | 11.522 4 | 11.790 | 11.866 | 11.894 | 11.849 | 11.930 8 | 13.029 | 12.799 | 12.818 | 12.798 | 12.806 16 | 15.649 | 14.373 | 14.353 | 14.404 | 14.384 32 | 22.462 | 17.733 | 17.861 | 17.908 | 17.830 64 | 44.593 | 26.238 | 26.461 | 26.512 | 26.578 128 | 135.504 | 47.256 | 47.394 | 47.372 | 47.537 256 | 818.483 | 105.361 | 105.749 | 105.619 | 105.772 ------------------------------------------------------------- Table 6: Speedup of query B (higher is better) -------------------------------------------------- n | v23 | ~v28-0005 | ~v28-0006 | ~v28-0007 -------------------------------------------------- 1 | 98.4% | 97.8% | 98.0% | 98.2% 2 | 98.8% | 98.2% | 98.3% | 98.4% 4 | 99.4% | 99.1% | 99.5% | 98.8% 8 | 101.8% | 101.6% | 101.8% | 101.7% 16 | 108.9% | 109.0% | 108.6% | 108.8% 32 | 126.7% | 125.8% | 125.4% | 126.0% 64 | 170.0% | 168.5% | 168.2% | 167.8% 128 | 286.7% | 285.9% | 286.0% | 285.0% 256 | 776.8% | 774.0% | 774.9% | 773.8% -------------------------------------------------- 4. Discussion First of all, tables 1, 2 and the figure attached to this email show that likely and unlikely do not have the effect I expected. Rather, tables 3, 4, 5 and 6 imply that they can have a negative effect on queries A and B. So it is better to remove these likely and unlikely. For the design change, the benchmark results show that it may cause some regression, especially for smaller sizes. However, Figure 1 also shows that the regression is much smaller than its variance. This design change is intended to improve code maintainability. The regression is small enough that I think these results are acceptable. What do you think about this? [1] https://www.postgresql.org/message-id/[email protected]... [2] https://www.postgresql.org/message-id/[email protected]... -- Best regards, Yuya Watari Attachments: [image/png] figure.png (234.9K, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/2-figure.png) download | view image [application/octet-stream] v28-0001-Speed-up-searches-for-child-EquivalenceMembers.patch (62.3K, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/3-v28-0001-Speed-up-searches-for-child-EquivalenceMembers.patch) download | inline diff: From f4299a4080a29e6ddf91ccb25eb5c96900aaf3c5 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:43:36 +0900 Subject: [PATCH v28 1/7] Speed up searches for child EquivalenceMembers Traditionally, child EquivalenceMembers were in the EquivalenceClass->ec_members. When we wanted to find some child members matching a request, we had to perform a linear search. This search became heavy when tables had many partitions, leading to much planning time. After this commit, child EquivalenceMembers no longer exist in ec_members. Instead, RelOptInfos have them. This change demonstrates a significant performance improvement in planning time. --- contrib/postgres_fdw/postgres_fdw.c | 36 +- src/backend/optimizer/path/equivclass.c | 464 ++++++++++++++++++++---- src/backend/optimizer/path/indxpath.c | 47 ++- src/backend/optimizer/path/pathkeys.c | 9 +- src/backend/optimizer/plan/createplan.c | 61 ++-- src/backend/optimizer/util/inherit.c | 14 + src/backend/optimizer/util/relnode.c | 89 +++++ src/include/nodes/pathnodes.h | 61 ++++ src/include/optimizer/pathnode.h | 18 + src/include/optimizer/paths.h | 12 +- src/tools/pgindent/typedefs.list | 1 + 11 files changed, 698 insertions(+), 114 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index c0810fbd7c8..e68c116164c 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7828,13 +7828,35 @@ conversion_error_callback(void *arg) EquivalenceMember * find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) { - ListCell *lc; - PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) rel->fdw_private; + Relids top_parent_rel_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; + + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); - foreach(lc, ec->ec_members) + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, rel->relids); /* * Note we require !bms_is_empty, else we'd accept constant @@ -7846,6 +7868,7 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) is_foreign_expr(root, rel, em->em_expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -7899,9 +7922,8 @@ find_em_for_rel_target(PlannerInfo *root, EquivalenceClass *ec, if (em->em_is_const) continue; - /* Ignore child members */ - if (em->em_is_child) - continue; + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* Match if same expression (after stripping relabel) */ em_expr = em->em_expr; diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index fae137dd825..25ce4dd2242 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -33,10 +33,14 @@ #include "utils/lsyscache.h" +static EquivalenceMember *make_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + EquivalenceMember *parent, + Oid datatype); static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, - EquivalenceMember *parent, Oid datatype); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); @@ -68,6 +72,11 @@ static bool reconsider_outer_join_clause(PlannerInfo *root, static bool reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo); static JoinDomain *find_join_domain(PlannerInfo *root, Relids relids); +static void add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel); static Bitmapset *get_eclass_indexes_for_relids(PlannerInfo *root, Relids relids); static Bitmapset *get_common_eclass_indexes(PlannerInfo *root, Relids relids1, @@ -373,7 +382,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); @@ -390,7 +399,7 @@ process_equivalence(PlannerInfo *root, { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); @@ -422,9 +431,9 @@ process_equivalence(PlannerInfo *root, ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; em1 = add_eq_member(ec, item1, item1_relids, - jdomain, NULL, item1_type); + jdomain, item1_type); em2 = add_eq_member(ec, item2, item2_relids, - jdomain, NULL, item2_type); + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -510,11 +519,16 @@ canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation) } /* - * add_eq_member - build a new EquivalenceMember and add it to an EC + * make_eq_member + * + * Build a new EquivalenceMember without adding it to an EC. If 'parent' + * parameter is NULL, the result will be a parent member, otherwise a child + * member. Note that child EquivalenceMembers should not be added to its + * parent EquivalenceClass. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) +make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype) { EquivalenceMember *em = makeNode(EquivalenceMember); @@ -525,6 +539,8 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, em->em_datatype = datatype; em->em_jdomain = jdomain; em->em_parent = parent; + em->em_child_relids = NULL; + em->em_child_joinrel_relids = NULL; if (bms_is_empty(relids)) { @@ -545,11 +561,29 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, { ec->ec_relids = bms_add_members(ec->ec_relids, relids); } - ec->ec_members = lappend(ec->ec_members, em); return em; } +/* + * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * + * Note: We don't have a function to add a child member like + * add_child_eq_member() because how to do it depends on the relations they are + * translated from. See add_child_rel_equivalences() and + * add_child_join_rel_equivalences() to see how to add child members. + */ +static EquivalenceMember * +add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) +{ + EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, + NULL, datatype); + + ec->ec_members = lappend(ec->ec_members, em); + return em; +} + /* * get_eclass_for_sort_expr @@ -598,6 +632,17 @@ get_eclass_for_sort_expr(PlannerInfo *root, EquivalenceMember *newem; ListCell *lc1; MemoryContext oldcontext; + Relids top_parent_rel; + + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_rel = find_relids_top_parents(root, rel); /* * Ensure the expression exposes the correct type and collation. @@ -616,7 +661,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, foreach(lc1, root->eq_classes) { EquivalenceClass *cur_ec = (EquivalenceClass *) lfirst(lc1); - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; /* * Never match to a volatile EC, except when we are looking at another @@ -631,16 +677,28 @@ get_eclass_for_sort_expr(PlannerInfo *root, if (!equal(opfamilies, cur_ec->ec_opfamilies)) continue; - foreach(lc2, cur_ec->ec_members) + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc2); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel); /* - * Ignore child members unless they match the request. + * Ignore child members unless they match the request. If this + * EquivalenceMember is a child, i.e., translated above, it should + * match the request. We cannot assert this if a request is + * bms_is_subset(). */ - if (cur_em->em_is_child && - !bms_equal(cur_em->em_relids, rel)) - continue; + Assert(!cur_em->em_is_child || bms_equal(cur_em->em_relids, rel)); /* * Match constants only within the same JoinDomain (see @@ -653,6 +711,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, equal(expr, cur_em->em_expr)) return cur_ec; /* Match! */ } + dispose_eclass_child_member_iterator(&it); } /* No match; does caller want a NULL result? */ @@ -690,7 +749,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, expr_relids = pull_varnos(root, (Node *) expr); newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, NULL, opcintype); + jdomain, opcintype); /* * add_eq_member doesn't check for volatile functions, set-returning @@ -760,19 +819,35 @@ get_eclass_for_sort_expr(PlannerInfo *root, * Child EC members are ignored unless they belong to given 'relids'. */ EquivalenceMember * -find_ec_member_matching_expr(EquivalenceClass *ec, +find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, Expr *expr, Relids relids) { - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; + + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_relids = find_relids_top_parents(root, relids); /* We ignore binary-compatible relabeling on both ends */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; - foreach(lc, ec->ec_members) + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); Expr *emexpr; /* @@ -782,6 +857,14 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (em->em_is_const) continue; + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -799,6 +882,7 @@ find_ec_member_matching_expr(EquivalenceClass *ec, if (equal(emexpr, expr)) return em; } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -841,7 +925,9 @@ find_computable_ec_member(PlannerInfo *root, bool require_parallel_safe) { List *exprvars; - ListCell *lc; + Relids top_parent_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *em; /* * Pull out the Vars and quasi-Vars present in "exprs". In the typical @@ -854,9 +940,23 @@ find_computable_ec_member(PlannerInfo *root, PVC_INCLUDE_WINDOWFUNCS | PVC_INCLUDE_PLACEHOLDERS); - foreach(lc, ec->ec_members) + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_relids = find_relids_top_parents(root, relids); + + /* + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. + */ + setup_eclass_child_member_iterator(&it, ec); + while ((em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *em = (EquivalenceMember *) lfirst(lc); List *emvars; ListCell *lc2; @@ -867,6 +967,14 @@ find_computable_ec_member(PlannerInfo *root, if (em->em_is_const) continue; + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + bms_is_subset(em->em_relids, top_parent_relids)) + iterate_child_rel_equivalences(&it, root, ec, em, relids); + /* * Ignore child members unless they belong to the requested rel. */ @@ -900,6 +1008,7 @@ find_computable_ec_member(PlannerInfo *root, return em; /* found usable expression */ } + dispose_eclass_child_member_iterator(&it); return NULL; } @@ -938,7 +1047,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, { Expr *targetexpr = (Expr *) lfirst(lc); - em = find_ec_member_matching_expr(ec, targetexpr, rel->relids); + em = find_ec_member_matching_expr(root, ec, targetexpr, rel->relids); if (!em) continue; @@ -1563,7 +1672,19 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *new_members = NIL; List *outer_members = NIL; List *inner_members = NIL; - ListCell *lc1; + Relids top_parent_join_relids; + EquivalenceChildMemberIterator it; + EquivalenceMember *cur_em; + + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_join_relids = find_relids_top_parents(root, join_relids); /* * First, scan the EC to identify member values that are computable at the @@ -1573,10 +1694,20 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * enforce that any newly computable members are all equal to each other * as well as to at least one input member, plus enforce at least one * outer-rel member equal to at least one inner-rel member. + * + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. */ - foreach(lc1, ec->ec_members) + setup_eclass_child_member_iterator(&it, ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + bms_is_subset(cur_em->em_relids, top_parent_join_relids)) + iterate_child_rel_equivalences(&it, root, ec, cur_em, join_relids); /* * We don't need to check explicitly for child EC members. This test @@ -1593,6 +1724,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, else new_members = lappend(new_members, cur_em); } + dispose_eclass_child_member_iterator(&it); /* * First, select the joinclause if needed. We can equate any one outer @@ -1610,6 +1742,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, Oid best_eq_op = InvalidOid; int best_score = -1; RestrictInfo *rinfo; + ListCell *lc1; foreach(lc1, outer_members) { @@ -1684,6 +1817,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, List *old_members = list_concat(outer_members, inner_members); EquivalenceMember *prev_em = NULL; RestrictInfo *rinfo; + ListCell *lc1; /* For now, arbitrarily take the first old_member as the one to use */ if (old_members) @@ -1691,7 +1825,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, foreach(lc1, new_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc1); + cur_em = (EquivalenceMember *) lfirst(lc1); if (prev_em != NULL) { @@ -2404,6 +2538,7 @@ reconsider_full_join_clause(PlannerInfo *root, OuterJoinClauseInfo *ojcinfo) if (matchleft && matchright) { cur_ec->ec_members = list_delete_nth_cell(cur_ec->ec_members, coal_idx); + Assert(!bms_is_empty(coal_em->em_relids)); return true; } @@ -2525,8 +2660,8 @@ exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, Oid opfamily) { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); if (equal(item1, em->em_expr)) item1member = true; else if (equal(item2, em->em_expr)) @@ -2597,8 +2732,8 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); Var *var; - if (em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); /* EM must be a Var, possibly with RelabelType */ var = (Var *) em->em_expr; @@ -2695,6 +2830,7 @@ add_child_rel_equivalences(PlannerInfo *root, Relids top_parent_relids = child_rel->top_parent_relids; Relids child_relids = child_rel->relids; int i; + ListCell *lc; /* * EC merging should be complete already, so we can use the parent rel's @@ -2707,7 +2843,6 @@ add_child_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(parent_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2720,15 +2855,9 @@ add_child_rel_equivalences(PlannerInfo *root, /* Sanity check eclass_indexes only contain ECs for parent_rel */ Assert(bms_is_subset(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2741,8 +2870,8 @@ add_child_rel_equivalences(PlannerInfo *root, * combinations of children. (But add_child_join_rel_equivalences * may add targeted combinations for partitionwise-join purposes.) */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * Consider only members that reference and can be computed at @@ -2758,6 +2887,7 @@ add_child_rel_equivalences(PlannerInfo *root, /* OK, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_rel->reloptkind == RELOPT_BASEREL) { @@ -2787,9 +2917,20 @@ add_child_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_rel->eclass_child_members = lappend(child_rel->eclass_child_members, + child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_relids = bms_add_member(cur_em->em_child_relids, + child_rel->relid); /* Record this EC index for the child rel */ child_rel->eclass_indexes = bms_add_member(child_rel->eclass_indexes, i); @@ -2818,6 +2959,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, Relids child_relids = child_joinrel->relids; Bitmapset *matching_ecs; MemoryContext oldcontext; + ListCell *lc; int i; Assert(IS_JOIN_REL(child_joinrel) && IS_JOIN_REL(parent_joinrel)); @@ -2839,7 +2981,6 @@ add_child_join_rel_equivalences(PlannerInfo *root, while ((i = bms_next_member(matching_ecs, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); - int num_members; /* * If this EC contains a volatile expression, then generating child @@ -2852,15 +2993,9 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Sanity check on get_eclass_indexes_for_relids result */ Assert(bms_overlap(top_parent_relids, cur_ec->ec_relids)); - /* - * We don't use foreach() here because there's no point in scanning - * newly-added child members, so we can stop after the last - * pre-existing EC member. - */ - num_members = list_length(cur_ec->ec_members); - for (int pos = 0; pos < num_members; pos++) + foreach(lc, cur_ec->ec_members) { - EquivalenceMember *cur_em = (EquivalenceMember *) list_nth(cur_ec->ec_members, pos); + EquivalenceMember *cur_em = lfirst_node(EquivalenceMember, lc); if (cur_em->em_is_const) continue; /* ignore consts here */ @@ -2869,8 +3004,8 @@ add_child_join_rel_equivalences(PlannerInfo *root, * We consider only original EC members here, not * already-transformed child members. */ - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); /* * We may ignore expressions that reference a single baserel, @@ -2885,6 +3020,7 @@ add_child_join_rel_equivalences(PlannerInfo *root, /* Yes, generate transformed child version */ Expr *child_expr; Relids new_relids; + EquivalenceMember *child_em; if (parent_joinrel->reloptkind == RELOPT_JOINREL) { @@ -2915,9 +3051,21 @@ add_child_join_rel_equivalences(PlannerInfo *root, top_parent_relids); new_relids = bms_add_members(new_relids, child_relids); - (void) add_eq_member(cur_ec, child_expr, new_relids, - cur_em->em_jdomain, - cur_em, cur_em->em_datatype); + child_em = make_eq_member(cur_ec, child_expr, new_relids, + cur_em->em_jdomain, + cur_em, cur_em->em_datatype); + child_joinrel->eclass_child_members = + lappend(child_joinrel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated + * using 'child_joinrel'. This knowledge is useful for + * add_transformed_child_version() to find child members from + * the given Relids. + */ + cur_em->em_child_joinrel_relids = + bms_add_member(cur_em->em_child_joinrel_relids, + child_joinrel->join_rel_list_index); } } } @@ -2986,6 +3134,161 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, list_length(root->eq_classes) - 1); } +/* + * setup_eclass_child_member_iterator + * Setup an EquivalenceChildMemberIterator 'it' so that it can iterate over + * EquivalenceMembers in 'ec'. + * + * Once used, the caller should dispose of the iterator by calling + * dispose_eclass_child_member_iterator(). + */ +void +setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec) +{ + it->index = -1; + it->modified = false; + it->ec_members = ec->ec_members; +} + +/* + * eclass_child_member_iterator_next + * Get a next EquivalenceMember from an EquivalenceChildMemberIterator 'it' + * that was setup by setup_eclass_child_member_iterator(). NULL is returned + * if there are no members left. + */ +EquivalenceMember * +eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it) +{ + if (++it->index < list_length(it->ec_members)) + return list_nth_node(EquivalenceMember, it->ec_members, it->index); + return NULL; +} + +/* + * dispose_eclass_child_member_iterator + * Free any memory allocated by the iterator. + */ +void +dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it) +{ + /* + * XXX Should we use list_free()? I decided to use this style to take + * advantage of speculative execution. + */ + if (unlikely(it->modified)) + pfree(it->ec_members); +} + +/* + * add_transformed_child_version + * Add a transformed EquivalenceMember referencing the given child_rel to + * the iterator. + * + * This function is expected to be called only from + * iterate_child_rel_equivalences(). + */ +static void +add_transformed_child_version(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + RelOptInfo *child_rel) +{ + ListCell *lc; + + foreach(lc, child_rel->eclass_child_members) + { + EquivalenceMember *child_em = lfirst_node(EquivalenceMember, lc); + + /* Skip unwanted EquivalenceMembers */ + if (child_em->em_parent != parent_em) + continue; + + /* + * If this is the first time the iterator's list has been modified, we + * need to make a copy of it. + */ + if (!it->modified) + { + it->ec_members = list_copy(it->ec_members); + it->modified = true; + } + + /* Add this child EquivalenceMember to the list */ + it->ec_members = lappend(it->ec_members, child_em); + } +} + +/* + * iterate_child_rel_equivalences + * Add transformed EquivalenceMembers referencing child rels in the given + * child_relids to the iterator. + * + * The transformation is done in add_transformed_child_version(). + */ +void +iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids) +{ + int i; + Relids matching_relids; + + /* The given EquivalenceMember should be parent */ + Assert(!parent_em->em_is_child); + + /* + * EquivalenceClass->ec_members has only parent EquivalenceMembers. Child + * members are translated using child RelOptInfos and stored in them. This + * is done in add_child_rel_equivalences() and + * add_child_join_rel_equivalences(). To retrieve child EquivalenceMembers + * of some parent, we need to know which RelOptInfos have such child + * members. This information is stored in indexes named em_child_relids + * and em_child_joinrel_relids. + * + * This function iterates over the child EquivalenceMembers that reference + * the given 'child_relids'. To do this, we first intersect 'child_relids' + * with these indexes. The result contains Relids of RelOptInfos that have + * child EquivalenceMembers we want to retrieve. Then we get the child + * members from the RelOptInfos using add_transformed_child_version(). + * + * We need to do these steps for each of the two indexes. + */ + + /* + * First, we translate simple rels. + */ + matching_relids = bms_intersect(parent_em->em_child_relids, + child_relids); + i = -1; + while ((i = bms_next_member(matching_relids, i)) >= 0) + { + /* Add transformed child version for this child rel */ + RelOptInfo *child_rel = root->simple_rel_array[i]; + + Assert(child_rel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_rel); + } + bms_free(matching_relids); + + /* + * Next, we try to translate join rels. + */ + i = -1; + while ((i = bms_next_member(parent_em->em_child_joinrel_relids, i)) >= 0) + { + /* Add transformed child version for this child join rel */ + RelOptInfo *child_joinrel = + list_nth_node(RelOptInfo, root->join_rel_list, i); + + Assert(child_joinrel != NULL); + add_transformed_child_version(it, root, ec, parent_em, child_joinrel); + } +} + /* * generate_implied_equalities_for_column @@ -3019,7 +3322,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, { List *result = NIL; bool is_child_rel = (rel->reloptkind == RELOPT_OTHER_MEMBER_REL); - Relids parent_relids; + Relids parent_relids, + top_parent_rel_relids; int i; /* Should be OK to rely on eclass_indexes */ @@ -3028,6 +3332,16 @@ generate_implied_equalities_for_column(PlannerInfo *root, /* Indexes are available only on base or "other" member relations. */ Assert(IS_SIMPLE_REL(rel)); + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_rel_relids = find_relids_top_parents(root, rel->relids); + /* If it's a child rel, we'll need to know what its parent(s) are */ if (is_child_rel) parent_relids = find_childrel_parents(root, rel); @@ -3040,6 +3354,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); EquivalenceMember *cur_em; ListCell *lc2; + EquivalenceChildMemberIterator it; /* Sanity check eclass_indexes only contain ECs for rel */ Assert(is_child_rel || bms_is_subset(rel->relids, cur_ec->ec_relids)); @@ -3060,16 +3375,25 @@ generate_implied_equalities_for_column(PlannerInfo *root, * column gets matched to. This is annoying but it only happens in * corner cases, so for now we live with just reporting the first * match. See also get_eclass_for_sort_expr.) + * + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. */ - cur_em = NULL; - foreach(lc2, cur_ec->ec_members) + setup_eclass_child_member_iterator(&it, cur_ec); + while ((cur_em = eclass_child_member_iterator_next(&it)) != NULL) { - cur_em = (EquivalenceMember *) lfirst(lc2); + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + bms_equal(cur_em->em_relids, top_parent_rel_relids)) + iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel->relids); if (bms_equal(cur_em->em_relids, rel->relids) && callback(root, rel, cur_ec, cur_em, callback_arg)) break; - cur_em = NULL; } + dispose_eclass_child_member_iterator(&it); if (!cur_em) continue; @@ -3084,8 +3408,8 @@ generate_implied_equalities_for_column(PlannerInfo *root, Oid eq_op; RestrictInfo *rinfo; - if (other_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!other_em->em_is_child); /* Make sure it'll be a join to a different rel */ if (other_em == cur_em || @@ -3303,8 +3627,8 @@ eclass_useful_for_merging(PlannerInfo *root, { EquivalenceMember *cur_em = (EquivalenceMember *) lfirst(lc); - if (cur_em->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!cur_em->em_is_child); if (!bms_overlap(cur_em->em_relids, relids)) return true; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 5d102a0d371..4a42752486e 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -190,7 +190,7 @@ static IndexClause *expand_indexqual_rowcompare(PlannerInfo *root, IndexOptInfo *index, Oid expr_op, bool var_on_left); -static void match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +static void match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p); static Expr *match_clause_to_ordering_op(IndexOptInfo *index, @@ -934,7 +934,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, * query_pathkeys will allow an incremental sort to be considered on * the index's partially sorted results. */ - match_pathkeys_to_index(index, root->query_pathkeys, + match_pathkeys_to_index(root, index, root->query_pathkeys, &orderbyclauses, &orderbyclausecols); if (list_length(root->query_pathkeys) == list_length(orderbyclauses)) @@ -3726,12 +3726,23 @@ expand_indexqual_rowcompare(PlannerInfo *root, * item in the given 'pathkeys' list. */ static void -match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, +match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, List **orderby_clauses_p, List **clause_columns_p) { + Relids top_parent_index_relids; ListCell *lc1; + /* + * First, we translate the given Relids to their top-level parents. This + * is required because an EquivalenceClass contains only parent + * EquivalenceMembers, and we have to translate top-level ones to get + * child members. We can skip such translations if we now see top-level + * ones, i.e., when top_parent_rel is NULL. See the + * find_relids_top_parents()'s definition for more details. + */ + top_parent_index_relids = find_relids_top_parents(root, index->rel->relids); + *orderby_clauses_p = NIL; /* set default results */ *clause_columns_p = NIL; @@ -3743,7 +3754,8 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, { PathKey *pathkey = (PathKey *) lfirst(lc1); bool found = false; - ListCell *lc2; + EquivalenceChildMemberIterator it; + EquivalenceMember *member; /* Pathkey must request default sort order for the target opfamily */ @@ -3762,16 +3774,36 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, * child member of more than one EC. Therefore, the same index could * be considered to match more than one pathkey list, which is OK * here. See also get_eclass_for_sort_expr.) + * + * If we need to see child EquivalenceMembers, we access them via + * EquivalenceChildMemberIterator during the iteration. */ - foreach(lc2, pathkey->pk_eclass->ec_members) + setup_eclass_child_member_iterator(&it, pathkey->pk_eclass); + while ((member = eclass_child_member_iterator_next(&it))) { - EquivalenceMember *member = (EquivalenceMember *) lfirst(lc2); int indexcol; + /* + * If child EquivalenceMembers may match the request, we add and + * iterate over them by calling iterate_child_rel_equivalences(). + */ + if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + bms_equal(member->em_relids, top_parent_index_relids)) + iterate_child_rel_equivalences(&it, root, pathkey->pk_eclass, member, + index->rel->relids); + /* No possibility of match if it references other relations */ - if (!bms_equal(member->em_relids, index->rel->relids)) + if (!member->em_is_child && + !bms_equal(member->em_relids, index->rel->relids)) continue; + /* + * If this EquivalenceMember is a child, i.e., translated above, + * it should match the request. We cannot assert this if a request + * is bms_is_subset(). + */ + Assert(bms_equal(member->em_relids, index->rel->relids)); + /* * We allow any column of the index to match each pathkey; they * don't have to match left-to-right as you might expect. This is @@ -3800,6 +3832,7 @@ match_pathkeys_to_index(IndexOptInfo *index, List *pathkeys, if (found) /* don't want to look at remaining members */ break; } + dispose_eclass_child_member_iterator(&it); /* * Return the matches found so far when this pathkey couldn't be diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 03c2ac36e05..b8f9c5f0e93 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -1151,8 +1151,8 @@ convert_subquery_pathkeys(PlannerInfo *root, RelOptInfo *rel, Oid sub_expr_coll = sub_eclass->ec_collation; ListCell *k; - if (sub_member->em_is_child) - continue; /* ignore children here */ + /* Child members should not exist in ec_members */ + Assert(!sub_member->em_is_child); foreach(k, subquery_tlist) { @@ -1709,8 +1709,11 @@ select_outer_pathkeys_for_merge(PlannerInfo *root, { EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2); + /* Child members should not exist in ec_members */ + Assert(!em->em_is_child); + /* Potential future join partner? */ - if (!em->em_is_const && !em->em_is_child && + if (!em->em_is_const && !bms_overlap(em->em_relids, joinrel->relids)) score++; } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 178c572b021..b5f8d5fad0a 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -264,7 +264,9 @@ static IncrementalSort *make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); -static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Plan *prepare_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -273,9 +275,11 @@ static Plan *prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); -static Sort *make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +static Sort *make_sort_from_pathkeys(PlannerInfo *root, + Plan *lefttree, + List *pathkeys, Relids relids); -static IncrementalSort *make_incrementalsort_from_pathkeys(Plan *lefttree, +static IncrementalSort *make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, int nPresortedCols); static Sort *make_sort_from_groupcols(List *groupcls, AttrNumber *grpColIdx, @@ -297,7 +301,7 @@ static Group *make_group(List *tlist, List *qual, int numGroupCols, AttrNumber *grpColIdx, Oid *grpOperators, Oid *grpCollations, Plan *lefttree); static Unique *make_unique_from_sortclauses(Plan *lefttree, List *distinctList); -static Unique *make_unique_from_pathkeys(Plan *lefttree, +static Unique *make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, int numCols); static Gather *make_gather(List *qptlist, List *qpqual, int nworkers, int rescan_param, bool single_copy, Plan *subplan); @@ -1285,7 +1289,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * function result; it must be the same plan node. However, we then * need to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys((Plan *) plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, (Plan *) plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1329,7 +1333,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) * don't need an explicit sort, to make sure they are returning * the same sort key columns the Append expects. */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, nodeSortColIdx, false, @@ -1470,7 +1474,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * function result; it must be the same plan node. However, we then need * to detect whether any tlist entries were added. */ - (void) prepare_sort_from_pathkeys(plan, pathkeys, + (void) prepare_sort_from_pathkeys(root, plan, pathkeys, best_path->path.parent->relids, NULL, true, @@ -1501,7 +1505,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, subplan = create_plan_recurse(root, subpath, CP_EXACT_TLIST); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, subpath->parent->relids, node->sortColIdx, false, @@ -1981,7 +1985,7 @@ create_gather_merge_plan(PlannerInfo *root, GatherMergePath *best_path) Assert(pathkeys != NIL); /* Compute sort column info, and adjust subplan's tlist as needed */ - subplan = prepare_sort_from_pathkeys(subplan, pathkeys, + subplan = prepare_sort_from_pathkeys(root, subplan, pathkeys, best_path->subpath->parent->relids, gm_plan->sortColIdx, false, @@ -2195,7 +2199,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) * relids. Thus, if this sort path is based on a child relation, we must * pass its relids. */ - plan = make_sort_from_pathkeys(subplan, best_path->path.pathkeys, + plan = make_sort_from_pathkeys(root, subplan, best_path->path.pathkeys, IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -2219,7 +2223,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, /* See comments in create_sort_plan() above */ subplan = create_plan_recurse(root, best_path->spath.subpath, flags | CP_SMALL_TLIST); - plan = make_incrementalsort_from_pathkeys(subplan, + plan = make_incrementalsort_from_pathkeys(root, subplan, best_path->spath.path.pathkeys, IS_OTHER_REL(best_path->spath.subpath->parent) ? best_path->spath.path.parent->relids : NULL, @@ -2288,7 +2292,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag subplan = create_plan_recurse(root, best_path->subpath, flags | CP_LABEL_TLIST); - plan = make_unique_from_pathkeys(subplan, + plan = make_unique_from_pathkeys(root, subplan, best_path->path.pathkeys, best_path->numkeys); @@ -4550,7 +4554,8 @@ create_mergejoin_plan(PlannerInfo *root, if (!use_incremental_sort) { sort_plan = (Plan *) - make_sort_from_pathkeys(outer_plan, + make_sort_from_pathkeys(root, + outer_plan, best_path->outersortkeys, outer_relids); @@ -4559,7 +4564,8 @@ create_mergejoin_plan(PlannerInfo *root, else { sort_plan = (Plan *) - make_incrementalsort_from_pathkeys(outer_plan, + make_incrementalsort_from_pathkeys(root, + outer_plan, best_path->outersortkeys, outer_relids, presorted_keys); @@ -4584,7 +4590,7 @@ create_mergejoin_plan(PlannerInfo *root, */ Relids inner_relids = inner_path->parent->relids; - Sort *sort = make_sort_from_pathkeys(inner_plan, + Sort *sort = make_sort_from_pathkeys(root, inner_plan, best_path->innersortkeys, inner_relids); @@ -6234,7 +6240,7 @@ make_incrementalsort(Plan *lefttree, int numCols, int nPresortedCols, * or a Result stacked atop lefttree). */ static Plan * -prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, +prepare_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, @@ -6301,7 +6307,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ @@ -6329,7 +6335,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, foreach(j, tlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, relids); + em = find_ec_member_matching_expr(root, ec, tle->expr, relids); if (em) { /* found expr already in tlist */ @@ -6345,7 +6351,7 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, /* * No matching tlist item; look for a computable expression. */ - em = find_computable_ec_member(NULL, ec, tlist, relids, false); + em = find_computable_ec_member(root, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; @@ -6416,7 +6422,8 @@ prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ static Sort * -make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) +make_sort_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + Relids relids) { int numsortkeys; AttrNumber *sortColIdx; @@ -6425,7 +6432,7 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6451,8 +6458,9 @@ make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) * 'nPresortedCols' is the number of presorted columns in input tuples */ static IncrementalSort * -make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, - Relids relids, int nPresortedCols) +make_incrementalsort_from_pathkeys(PlannerInfo *root, Plan *lefttree, + List *pathkeys, Relids relids, + int nPresortedCols) { int numsortkeys; AttrNumber *sortColIdx; @@ -6461,7 +6469,7 @@ make_incrementalsort_from_pathkeys(Plan *lefttree, List *pathkeys, bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ - lefttree = prepare_sort_from_pathkeys(lefttree, pathkeys, + lefttree = prepare_sort_from_pathkeys(root, lefttree, pathkeys, relids, NULL, false, @@ -6820,7 +6828,8 @@ make_unique_from_sortclauses(Plan *lefttree, List *distinctList) * as above, but use pathkeys to identify the sort columns and semantics */ static Unique * -make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) +make_unique_from_pathkeys(PlannerInfo *root, Plan *lefttree, List *pathkeys, + int numCols) { Unique *node = makeNode(Unique); Plan *plan = &node->plan; @@ -6883,7 +6892,7 @@ make_unique_from_pathkeys(Plan *lefttree, List *pathkeys, int numCols) foreach(j, plan->targetlist) { tle = (TargetEntry *) lfirst(j); - em = find_ec_member_matching_expr(ec, tle->expr, NULL); + em = find_ec_member_matching_expr(root, ec, tle->expr, NULL); if (em) { /* found expr already in tlist */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index c5b906a9d43..3c2198ea5bd 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -470,6 +470,7 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, RelationGetRelid(parentrel); Oid childOID = RelationGetRelid(childrel); RangeTblEntry *childrte; + Index topParentRTindex; Index childRTindex; AppendRelInfo *appinfo; TupleDesc child_tupdesc; @@ -577,6 +578,19 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, Assert(root->append_rel_array[childRTindex] == NULL); root->append_rel_array[childRTindex] = appinfo; + /* + * Find a top parent rel's index and save it to top_parent_relid_array. + */ + if (root->top_parent_relid_array == NULL) + root->top_parent_relid_array = + (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); + Assert(root->top_parent_relid_array[childRTindex] == 0); + topParentRTindex = parentRTindex; + while (root->append_rel_array[topParentRTindex] != NULL && + root->append_rel_array[topParentRTindex]->parent_relid != 0) + topParentRTindex = root->append_rel_array[topParentRTindex]->parent_relid; + root->top_parent_relid_array[childRTindex] = topParentRTindex; + /* * Build a PlanRowMark if parent is marked FOR UPDATE/SHARE. */ diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index f96573eb5d6..ce494364173 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -123,11 +123,14 @@ setup_simple_rel_arrays(PlannerInfo *root) if (root->append_rel_list == NIL) { root->append_rel_array = NULL; + root->top_parent_relid_array = NULL; return; } root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); + root->top_parent_relid_array = (Index *) + palloc0(size * sizeof(Index)); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -148,6 +151,27 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array[child_relid] = appinfo; } + + /* + * Find a top parent rel's relid for each AppendRelInfo. We cannot do this + * in the last foreach loop because there may be multi-level parent-child + * relations. + */ + for (int i = 0; i < size; i++) + { + int top_parent_relid; + + if (root->append_rel_array[i] == NULL) + continue; + + top_parent_relid = root->append_rel_array[i]->parent_relid; + + while (root->append_rel_array[top_parent_relid] != NULL && + root->append_rel_array[top_parent_relid]->parent_relid != 0) + top_parent_relid = root->append_rel_array[top_parent_relid]->parent_relid; + + root->top_parent_relid_array[i] = top_parent_relid; + } } /* @@ -175,12 +199,25 @@ expand_planner_arrays(PlannerInfo *root, int add_size) repalloc0_array(root->simple_rte_array, RangeTblEntry *, root->simple_rel_array_size, new_size); if (root->append_rel_array) + { root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); + root->top_parent_relid_array = + repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + } else + { root->append_rel_array = palloc0_array(AppendRelInfo *, new_size); + /* + * We do not allocate top_parent_relid_array here so that + * find_relids_top_parents() can early find all of the given Relids + * are top-level. + */ + root->top_parent_relid_array = NULL; + } + root->simple_rel_array_size = new_size; } @@ -234,6 +271,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptInfo *parent) rel->subplan_params = NIL; rel->rel_parallel_workers = -1; /* set up in get_relation_info */ rel->amflags = 0; + rel->eclass_child_members = NIL; rel->serverid = InvalidOid; if (rte->rtekind == RTE_RELATION) { @@ -629,6 +667,12 @@ add_join_rel(PlannerInfo *root, RelOptInfo *joinrel) /* GEQO requires us to append the new joinrel to the end of the list! */ root->join_rel_list = lappend(root->join_rel_list, joinrel); + /* + * Store the index of this joinrel to use in + * add_child_join_rel_equivalences(). + */ + joinrel->join_rel_list_index = list_length(root->join_rel_list) - 1; + /* store it into the auxiliary hashtable if there is one. */ if (root->join_rel_hash) { @@ -741,6 +785,7 @@ build_join_rel(PlannerInfo *root, joinrel->subplan_params = NIL; joinrel->rel_parallel_workers = -1; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -928,6 +973,7 @@ build_child_join_rel(PlannerInfo *root, RelOptInfo *outer_rel, joinrel->subroot = NULL; joinrel->subplan_params = NIL; joinrel->amflags = 0; + joinrel->eclass_child_members = NIL; joinrel->serverid = InvalidOid; joinrel->userid = InvalidOid; joinrel->useridiscurrent = false; @@ -1490,6 +1536,7 @@ fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids) upperrel->cheapest_total_path = NULL; upperrel->cheapest_unique_path = NULL; upperrel->cheapest_parameterized_paths = NIL; + upperrel->eclass_child_members = NIL; /* XXX Is this required? */ root->upper_rels[kind] = lappend(root->upper_rels[kind], upperrel); @@ -1530,6 +1577,48 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } +/* + * find_relids_top_parents_slow + * Slow path of find_relids_top_parents() macro. + * + * Do not call this directly; use the macro instead. See the macro comment for + * more details. + */ +Relids +find_relids_top_parents_slow(PlannerInfo *root, Relids relids) +{ + Index *top_parent_relid_array = root->top_parent_relid_array; + Relids result; + bool is_top_parent; + int i; + + /* This function should be called in the slow path */ + Assert(top_parent_relid_array != NULL); + + result = NULL; + is_top_parent = true; + i = -1; + while ((i = bms_next_member(relids, i)) >= 0) + { + int top_parent_relid = (int) top_parent_relid_array[i]; + + if (top_parent_relid == 0) + top_parent_relid = i; /* 'i' has no parents, so add itself */ + else if (top_parent_relid != i) + is_top_parent = false; + result = bms_add_member(result, top_parent_relid); + } + + if (is_top_parent) + { + bms_free(result); + return NULL; + } + + return result; +} + + /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index add0f9e45fc..a8cc7b19ff2 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -248,6 +248,14 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * top_parent_relid_array is the same length as simple_rel_array and holds + * the top-level parent indexes of the corresponding rels within + * simple_rel_array. The element can be zero if the rel has no parents, + * i.e., is itself in a top-level. + */ + Index *top_parent_relid_array pg_node_attr(read_write_ignore); + /* * all_baserels is a Relids set of all base relids (but not joins or * "other" rels) in the query. This is computed in deconstruct_jointree. @@ -957,6 +965,17 @@ typedef struct RelOptInfo /* Bitmask of optional features supported by the table AM */ uint32 amflags; + /* + * information about a join rel + */ + /* index in root->join_rel_list of this rel */ + Index join_rel_list_index; + + /* + * EquivalenceMembers referencing this rel + */ + List *eclass_child_members; + /* * Information about foreign tables and foreign joins */ @@ -1374,6 +1393,12 @@ typedef struct JoinDomain * entry: consider SELECT random() AS a, random() AS b ... ORDER BY b,a. * So we record the SortGroupRef of the originating sort clause. * + * EquivalenceClass->ec_members can only have parent members, and child members + * are stored in RelOptInfos, from which those child members are translated. To + * lookup child EquivalenceMembers, we use EquivalenceChildMemberIterator. See + * its comment for usage. The approach to lookup child members quickly is + * described as iterate_child_rel_equivalences() comment. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1446,10 +1471,46 @@ typedef struct EquivalenceMember bool em_is_child; /* derived version for a child relation? */ Oid em_datatype; /* the "nominal type" used by the opfamily */ JoinDomain *em_jdomain; /* join domain containing the source clause */ + Relids em_child_relids; /* all relids of child rels */ + Bitmapset *em_child_joinrel_relids; /* indexes in root->join_rel_list + * of join rel children */ /* if em_is_child is true, this links to corresponding EM for top parent */ struct EquivalenceMember *em_parent pg_node_attr(read_write_ignore); } EquivalenceMember; +/* + * EquivalenceChildMemberIterator + * + * EquivalenceClass has only parent EquivalenceMembers, so we need to translate + * child members if necessary. EquivalenceChildMemberIterator provides a way to + * iterate over translated child members during the loop in addition to all of + * the parent EquivalenceMembers. + * + * The most common way to use this struct is as follows: + * ----- + * EquivalenceClass *ec = given; + * Relids rel = given; + * EquivalenceChildMemberIterator it; + * EquivalenceMember *em; + * + * setup_eclass_child_member_iterator(&it, ec); + * while ((em = eclass_child_member_iterator_next(&it)) != NULL) + * { + * use em ...; + * if (we need to iterate over child EquivalenceMembers) + * iterate_child_rel_equivalences(&it, root, ec, em, rel); + * use em ...; + * } + * dispose_eclass_child_member_iterator(&it); + * ----- + */ +typedef struct +{ + int index; /* current index within 'ec_members'. */ + bool modified; /* is 'ec_members' a newly allocated one? */ + List *ec_members; /* parent and child members */ +} EquivalenceChildMemberIterator; + /* * PathKeys * diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 1035e6560c1..5e79cf1f63b 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -332,6 +332,24 @@ extern Relids min_join_parameterization(PlannerInfo *root, extern RelOptInfo *fetch_upper_rel(PlannerInfo *root, UpperRelationKind kind, Relids relids); extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); + +/* + * find_relids_top_parents + * Compute the set of top-parent relids of rel. + * + * Replaces all Relids appearing in the given 'relids' as their top-level + * parents. The result will be NULL if and only if all of the given relids are + * top-level. + * + * The motivation for having this feature as a macro rather than a function is + * that Relids are top-level in most cases. We can quickly determine when + * root->top_parent_relid_array is NULL. + */ +#define find_relids_top_parents(root, relids) \ + (likely((root)->top_parent_relid_array == NULL) \ + ? NULL : find_relids_top_parents_slow(root, relids)) +extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); + extern ParamPathInfo *get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, Relids required_outer); diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 54869d44013..50812e3a5d8 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -137,7 +137,8 @@ extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Index sortref, Relids rel, bool create_it); -extern EquivalenceMember *find_ec_member_matching_expr(EquivalenceClass *ec, +extern EquivalenceMember *find_ec_member_matching_expr(PlannerInfo *root, + EquivalenceClass *ec, Expr *expr, Relids relids); extern EquivalenceMember *find_computable_ec_member(PlannerInfo *root, @@ -179,6 +180,15 @@ extern void add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, List *child_tlist, List *setop_pathkeys); +extern void setup_eclass_child_member_iterator(EquivalenceChildMemberIterator *it, + EquivalenceClass *ec); +extern EquivalenceMember *eclass_child_member_iterator_next(EquivalenceChildMemberIterator *it); +extern void dispose_eclass_child_member_iterator(EquivalenceChildMemberIterator *it); +extern void iterate_child_rel_equivalences(EquivalenceChildMemberIterator *it, + PlannerInfo *root, + EquivalenceClass *ec, + EquivalenceMember *parent_em, + Relids child_relids); extern List *generate_implied_equalities_for_column(PlannerInfo *root, RelOptInfo *rel, ec_matches_callback_type callback, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 2d4c870423a..32e4be0f47f 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -683,6 +683,7 @@ EphemeralNamedRelation EphemeralNamedRelationData EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData +EquivalenceChildMemberIterator EquivalenceClass EquivalenceMember ErrorContextCallback -- 2.45.2.windows.1 [application/octet-stream] v28-0002-Introduce-indexes-for-RestrictInfo.patch (42.9K, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/4-v28-0002-Introduce-indexes-for-RestrictInfo.patch) download | inline diff: From 10c482062464b565d3438d293180e5a852007571 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 25 Aug 2023 10:44:25 +0900 Subject: [PATCH v28 2/7] Introduce indexes for RestrictInfo This commit adds indexes to speed up searches for RestrictInfos. When there are many child partitions and we have to perform planning for join operations on the tables, we have to handle a large number of RestrictInfos, resulting in a long planning time. This commit adds ec_[source|derive]_indexes to speed up the search. We can use the indexes to filter out unwanted RestrictInfos, and improve the planning performance. Author: David Rowley <[email protected]> and me, and includes rebase by Alena Rybakina <[email protected]> [1]. [1] https://www.postgresql.org/message-id/72d292a1-06ff-432a-a803-af5053786444%40yandex.ru --- src/backend/nodes/outfuncs.c | 6 +- src/backend/nodes/readfuncs.c | 2 + src/backend/optimizer/path/costsize.c | 3 +- src/backend/optimizer/path/equivclass.c | 516 ++++++++++++++++++++-- src/backend/optimizer/plan/analyzejoins.c | 49 +- src/backend/optimizer/plan/planner.c | 2 + src/backend/optimizer/prep/prepjointree.c | 2 + src/backend/optimizer/util/appendinfo.c | 2 + src/backend/optimizer/util/inherit.c | 7 + src/backend/optimizer/util/restrictinfo.c | 5 + src/include/nodes/parsenodes.h | 6 + src/include/nodes/pathnodes.h | 41 +- src/include/optimizer/paths.h | 21 +- 13 files changed, 593 insertions(+), 69 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 9827cf16be4..bd6a79b57a7 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -466,8 +466,8 @@ _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) WRITE_NODE_FIELD(ec_opfamilies); WRITE_OID_FIELD(ec_collation); WRITE_NODE_FIELD(ec_members); - WRITE_NODE_FIELD(ec_sources); - WRITE_NODE_FIELD(ec_derives); + WRITE_BITMAPSET_FIELD(ec_source_indexes); + WRITE_BITMAPSET_FIELD(ec_derive_indexes); WRITE_BITMAPSET_FIELD(ec_relids); WRITE_BOOL_FIELD(ec_has_const); WRITE_BOOL_FIELD(ec_has_volatile); @@ -573,6 +573,8 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); + WRITE_BITMAPSET_FIELD(eclass_source_indexes); + WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index be5f19dd7f6..70864fc1b29 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -434,6 +434,8 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); + READ_BITMAPSET_FIELD(eclass_source_indexes); + READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index c36687aa4df..1ec10212eb8 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -5839,7 +5839,8 @@ get_foreign_key_join_selectivity(PlannerInfo *root, if (ec && ec->ec_has_const) { EquivalenceMember *em = fkinfo->fk_eclass_member[i]; - RestrictInfo *rinfo = find_derived_clause_for_ec_member(ec, + RestrictInfo *rinfo = find_derived_clause_for_ec_member(root, + ec, em); if (rinfo) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 25ce4dd2242..474231499e6 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -42,6 +42,10 @@ static EquivalenceMember *add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, JoinDomain *jdomain, Oid datatype); +static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); +static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, + RestrictInfo *rinfo); static void generate_base_implied_equalities_const(PlannerInfo *root, EquivalenceClass *ec); static void generate_base_implied_equalities_no_const(PlannerInfo *root, @@ -319,7 +323,6 @@ process_equivalence(PlannerInfo *root, /* If case 1, nothing to do, except add to sources */ if (ec1 == ec2) { - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -330,6 +333,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); return true; } @@ -350,8 +355,10 @@ process_equivalence(PlannerInfo *root, * be found. */ ec1->ec_members = list_concat(ec1->ec_members, ec2->ec_members); - ec1->ec_sources = list_concat(ec1->ec_sources, ec2->ec_sources); - ec1->ec_derives = list_concat(ec1->ec_derives, ec2->ec_derives); + ec1->ec_source_indexes = bms_join(ec1->ec_source_indexes, + ec2->ec_source_indexes); + ec1->ec_derive_indexes = bms_join(ec1->ec_derive_indexes, + ec2->ec_derive_indexes); ec1->ec_relids = bms_join(ec1->ec_relids, ec2->ec_relids); ec1->ec_has_const |= ec2->ec_has_const; /* can't need to set has_volatile */ @@ -363,10 +370,9 @@ process_equivalence(PlannerInfo *root, root->eq_classes = list_delete_nth_cell(root->eq_classes, ec2_idx); /* just to avoid debugging confusion w/ dangling pointers: */ ec2->ec_members = NIL; - ec2->ec_sources = NIL; - ec2->ec_derives = NIL; + ec2->ec_source_indexes = NULL; + ec2->ec_derive_indexes = NULL; ec2->ec_relids = NULL; - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -377,13 +383,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec1) { /* Case 3: add item2 to ec1 */ em2 = add_eq_member(ec1, item2, item2_relids, jdomain, item2_type); - ec1->ec_sources = lappend(ec1->ec_sources, restrictinfo); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -394,13 +401,14 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec1, restrictinfo); } else if (ec2) { /* Case 3: add item1 to ec2 */ em1 = add_eq_member(ec2, item1, item1_relids, jdomain, item1_type); - ec2->ec_sources = lappend(ec2->ec_sources, restrictinfo); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -411,6 +419,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec2, restrictinfo); } else { @@ -420,8 +430,8 @@ process_equivalence(PlannerInfo *root, ec->ec_opfamilies = opfamilies; ec->ec_collation = collation; ec->ec_members = NIL; - ec->ec_sources = list_make1(restrictinfo); - ec->ec_derives = NIL; + ec->ec_source_indexes = NULL; + ec->ec_derive_indexes = NULL; ec->ec_relids = NULL; ec->ec_has_const = false; ec->ec_has_volatile = false; @@ -443,6 +453,8 @@ process_equivalence(PlannerInfo *root, /* mark the RI as usable with this pair of EMs */ restrictinfo->left_em = em1; restrictinfo->right_em = em2; + + add_eq_source(root, ec, restrictinfo); } return true; @@ -584,6 +596,167 @@ add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, return em; } +/* + * add_eq_source - add 'rinfo' in eq_sources for this 'ec' + */ +static void +add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int source_idx = list_length(root->eq_sources); + int i; + + ec->ec_source_indexes = bms_add_member(ec->ec_source_indexes, source_idx); + root->eq_sources = lappend(root->eq_sources, rinfo); + Assert(rinfo->eq_sources_index == -1); + rinfo->eq_sources_index = source_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, + source_idx); + } +} + +/* + * add_eq_derive - add 'rinfo' in eq_derives for this 'ec' + */ +static void +add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) +{ + int derive_idx = list_length(root->eq_derives); + int i; + + ec->ec_derive_indexes = bms_add_member(ec->ec_derive_indexes, derive_idx); + root->eq_derives = lappend(root->eq_derives, rinfo); + Assert(rinfo->eq_derives_index == -1); + rinfo->eq_derives_index = derive_idx; + + i = -1; + while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, + derive_idx); + } +} + +/* + * update_clause_relids + * Update RestrictInfo->clause_relids with adjusting the relevant indexes. + * The clause_relids field must be updated through this function, not by + * setting the field directly. + */ +void +update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids) +{ + int i; + Relids removing_relids; + Relids adding_relids; +#ifdef USE_ASSERT_CHECKING + Relids common_relids; +#endif + + /* + * If there is nothing to do, we return immediately after setting the + * clause_relids field. + */ + if (rinfo->eq_sources_index == -1 && rinfo->eq_derives_index == -1) + { + rinfo->clause_relids = new_clause_relids; + return; + } + + /* + * Remove any references between this RestrictInfo and RangeTblEntries + * that are no longer relevant. + */ + removing_relids = bms_difference(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(removing_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_del_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_del_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(removing_relids); + + /* + * Add references between this RestrictInfo and RangeTblEntries that will + * be relevant. + */ + adding_relids = bms_difference(new_clause_relids, rinfo->clause_relids); + i = -1; + while ((i = bms_next_member(adding_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + { + Assert(!bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + rte->eclass_source_indexes = + bms_add_member(rte->eclass_source_indexes, + rinfo->eq_sources_index); + } + if (rinfo->eq_derives_index != -1) + { + Assert(!bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + rte->eclass_derive_indexes = + bms_add_member(rte->eclass_derive_indexes, + rinfo->eq_derives_index); + } + } + bms_free(adding_relids); + +#ifdef USE_ASSERT_CHECKING + + /* + * Verify that all indexes are set for common Relids. + */ + common_relids = bms_intersect(rinfo->clause_relids, new_clause_relids); + i = -1; + while ((i = bms_next_member(common_relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + if (rinfo->eq_sources_index != -1) + Assert(bms_is_member(rinfo->eq_sources_index, + rte->eclass_source_indexes)); + if (rinfo->eq_derives_index != -1) + Assert(bms_is_member(rinfo->eq_derives_index, + rte->eclass_derive_indexes)); + } + bms_free(common_relids); +#endif + + /* + * Since we have done everything to adjust the indexes, we can set the + * clause_relids field. + */ + rinfo->clause_relids = new_clause_relids; +} + /* * get_eclass_for_sort_expr @@ -729,8 +902,8 @@ get_eclass_for_sort_expr(PlannerInfo *root, newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = collation; newec->ec_members = NIL; - newec->ec_sources = NIL; - newec->ec_derives = NIL; + newec->ec_source_indexes = NULL; + newec->ec_derive_indexes = NULL; newec->ec_relids = NULL; newec->ec_has_const = false; newec->ec_has_volatile = contain_volatile_functions((Node *) expr); @@ -1134,7 +1307,7 @@ relation_can_be_sorted_early(PlannerInfo *root, RelOptInfo *rel, * scanning of the quals and before Path construction begins. * * We make no attempt to avoid generating duplicate RestrictInfos here: we - * don't search ec_sources or ec_derives for matches. It doesn't really + * don't search eq_sources or eq_derives for matches. It doesn't really * seem worth the trouble to do so. */ void @@ -1227,6 +1400,7 @@ generate_base_implied_equalities_const(PlannerInfo *root, { EquivalenceMember *const_em = NULL; ListCell *lc; + int i; /* * In the trivial case where we just had one "var = const" clause, push @@ -1236,9 +1410,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, * equivalent to the old one. */ if (list_length(ec->ec_members) == 2 && - list_length(ec->ec_sources) == 1) + bms_get_singleton_member(ec->ec_source_indexes, &i)) { - RestrictInfo *restrictinfo = (RestrictInfo *) linitial(ec->ec_sources); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, root->eq_sources, i); distribute_restrictinfo_to_rels(root, restrictinfo); return; @@ -1296,9 +1470,9 @@ generate_base_implied_equalities_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the correct - * markings for a mergejoinable clause, and save it in ec_derives. (We + * markings for a mergejoinable clause, and save it in eq_derives. (We * will not re-use such clauses directly, but selectivity estimation - * may consult the list later. Note that this use of ec_derives does + * may consult the list later. Note that this use of eq_derives does * not overlap with its use for join clauses, since we never generate * join clauses from an ec_has_const eclass.) */ @@ -1308,7 +1482,8 @@ generate_base_implied_equalities_const(PlannerInfo *root, rinfo->left_ec = rinfo->right_ec = ec; rinfo->left_em = cur_em; rinfo->right_em = const_em; - ec->ec_derives = lappend(ec->ec_derives, rinfo); + + add_eq_derive(root, ec, rinfo); } } } @@ -1374,7 +1549,7 @@ generate_base_implied_equalities_no_const(PlannerInfo *root, /* * If the clause didn't degenerate to a constant, fill in the * correct markings for a mergejoinable clause. We don't put it - * in ec_derives however; we don't currently need to re-find such + * in eq_derives however; we don't currently need to re-find such * clauses, and we don't want to clutter that list with non-join * clauses. */ @@ -1431,11 +1606,12 @@ static void generate_base_implied_equalities_broken(PlannerInfo *root, EquivalenceClass *ec) { - ListCell *lc; + int i = -1; - foreach(lc, ec->ec_sources) + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); if (ec->ec_has_const || bms_membership(restrictinfo->required_relids) != BMS_MULTIPLE) @@ -1476,11 +1652,11 @@ generate_base_implied_equalities_broken(PlannerInfo *root, * Because the same join clauses are likely to be needed multiple times as * we consider different join paths, we avoid generating multiple copies: * whenever we select a particular pair of EquivalenceMembers to join, - * we check to see if the pair matches any original clause (in ec_sources) - * or previously-built clause (in ec_derives). This saves memory and allows - * re-use of information cached in RestrictInfos. We also avoid generating - * commutative duplicates, i.e. if the algorithm selects "a.x = b.y" but - * we already have "b.y = a.x", we return the existing clause. + * we check to see if the pair matches any original clause in root's + * eq_sources or previously-built clause (in root's eq_derives). This saves + * memory and allows re-use of information cached in RestrictInfos. We also + * avoid generating commutative duplicates, i.e. if the algorithm selects + * "a.x = b.y" but we already have "b.y = a.x", we return the existing clause. * * If we are considering an outer join, sjinfo is the associated OJ info, * otherwise it can be NULL. @@ -1870,12 +2046,16 @@ generate_join_implied_equalities_broken(PlannerInfo *root, Relids nominal_inner_relids, RelOptInfo *inner_rel) { + Bitmapset *matching_es; List *result = NIL; - ListCell *lc; + int i; - foreach(lc, ec->ec_sources) + matching_es = get_ec_source_indexes(root, ec, nominal_join_relids); + i = -1; + while ((i = bms_next_member(matching_es, i)) >= 0) { - RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *restrictinfo = list_nth_node(RestrictInfo, + root->eq_sources, i); Relids clause_relids = restrictinfo->required_relids; if (bms_is_subset(clause_relids, nominal_join_relids) && @@ -1887,12 +2067,12 @@ generate_join_implied_equalities_broken(PlannerInfo *root, /* * If we have to translate, just brute-force apply adjust_appendrel_attrs * to all the RestrictInfos at once. This will result in returning - * RestrictInfos that are not listed in ec_derives, but there shouldn't be + * RestrictInfos that are not listed in eq_derives, but there shouldn't be * any duplication, and it's a sufficiently narrow corner case that we * shouldn't sweat too much over it anyway. * * Since inner_rel might be an indirect descendant of the baserel - * mentioned in the ec_sources clauses, we have to be prepared to apply + * mentioned in the eq_sources clauses, we have to be prepared to apply * multiple levels of Var translation. */ if (IS_OTHER_REL(inner_rel) && result != NIL) @@ -1954,10 +2134,11 @@ create_join_clause(PlannerInfo *root, EquivalenceMember *rightem, EquivalenceClass *parent_ec) { + Bitmapset *matches; RestrictInfo *rinfo; RestrictInfo *parent_rinfo = NULL; - ListCell *lc; MemoryContext oldcontext; + int i; /* * Search to see if we already built a RestrictInfo for this pair of @@ -1968,9 +2149,12 @@ create_join_clause(PlannerInfo *root, * it's not identical, it'd better have the same effects, or the operator * families we're using are broken. */ - foreach(lc, ec->ec_sources) + matches = bms_int_members(get_ec_source_indexes_strict(root, ec, leftem->em_relids), + get_ec_source_indexes_strict(root, ec, rightem->em_relids)); + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -1981,9 +2165,13 @@ create_join_clause(PlannerInfo *root, return rinfo; } - foreach(lc, ec->ec_derives) + matches = bms_int_members(get_ec_derive_indexes_strict(root, ec, leftem->em_relids), + get_ec_derive_indexes_strict(root, ec, rightem->em_relids)); + + i = -1; + while ((i = bms_next_member(matches, i)) >= 0) { - rinfo = (RestrictInfo *) lfirst(lc); + rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); if (rinfo->left_em == leftem && rinfo->right_em == rightem && rinfo->parent_ec == parent_ec) @@ -2056,7 +2244,7 @@ create_join_clause(PlannerInfo *root, rinfo->left_em = leftem; rinfo->right_em = rightem; /* and save it for possible re-use */ - ec->ec_derives = lappend(ec->ec_derives, rinfo); + add_eq_derive(root, ec, rinfo); MemoryContextSwitchTo(oldcontext); @@ -2782,16 +2970,19 @@ match_eclasses_to_foreign_key_col(PlannerInfo *root, * Returns NULL if no such clause can be found. */ RestrictInfo * -find_derived_clause_for_ec_member(EquivalenceClass *ec, +find_derived_clause_for_ec_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em) { - ListCell *lc; + int i; Assert(ec->ec_has_const); Assert(!em->em_is_const); - foreach(lc, ec->ec_derives) + + i = -1; + while ((i = bms_next_member(ec->ec_derive_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); /* * generate_base_implied_equalities_const will have put non-const @@ -3517,7 +3708,7 @@ have_relevant_eclass_joinclause(PlannerInfo *root, * surely be true if both of them overlap ec_relids.) * * Note we don't test ec_broken; if we did, we'd need a separate code - * path to look through ec_sources. Checking the membership anyway is + * path to look through eq_sources. Checking the membership anyway is * OK as a possibly-overoptimistic heuristic. * * We don't test ec_has_const either, even though a const eclass won't @@ -3605,7 +3796,7 @@ eclass_useful_for_merging(PlannerInfo *root, /* * Note we don't test ec_broken; if we did, we'd need a separate code path - * to look through ec_sources. Checking the members anyway is OK as a + * to look through eq_sources. Checking the members anyway is OK as a * possibly-overoptimistic heuristic. */ @@ -3762,3 +3953,240 @@ get_common_eclass_indexes(PlannerInfo *root, Relids relids1, Relids relids2) /* Calculate and return the common EC indexes, recycling the left input. */ return bms_int_members(rel1ecs, rel2ecs); } + +/* + * get_ec_source_indexes + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_esis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_esis, ec->ec_source_indexes); +} + +/* + * get_ec_source_indexes_strict + * Returns a Bitmapset with indexes into root->eq_sources for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *esis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + esis = bms_intersect(ec->ec_source_indexes, + rte->eclass_source_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + esis = bms_int_members(esis, rte->eclass_source_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(esis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return esis; +} + +/* + * get_ec_derive_indexes + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_overlap(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + * + * XXX is this function even needed? + */ +Bitmapset * +get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) +{ + Bitmapset *rel_edis = NULL; + int i = -1; + + while ((i = bms_next_member(relids, i)) >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(rel_edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_overlap(relids, rinfo->clause_relids)); + } +#endif + + /* bitwise-AND to leave only the ones for this EquivalenceClass */ + return bms_int_members(rel_edis, ec->ec_derive_indexes); +} + +/* + * get_ec_derive_indexes_strict + * Returns a Bitmapset with indexes into root->eq_derives for all + * RestrictInfos in 'ec' that have + * bms_is_subset(relids, rinfo->clause_relids) as true. + * + * Returns a newly allocated Bitmapset which the caller is free to modify. + */ +Bitmapset * +get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, + Relids relids) +{ + Bitmapset *edis = NULL; + int i = bms_next_member(relids, -1); + + if (i >= 0) + { + RangeTblEntry *rte = root->simple_rte_array[i]; + + /* + * bms_intersect to the first relation to try to keep the resulting + * Bitmapset as small as possible. This saves having to make a + * complete bms_copy() of one of them. One may contain significantly + * more words than the other. + */ + edis = bms_intersect(ec->ec_derive_indexes, + rte->eclass_derive_indexes); + + while ((i = bms_next_member(relids, i)) >= 0) + { + rte = root->simple_rte_array[i]; + edis = bms_int_members(edis, rte->eclass_derive_indexes); + } + } + +#ifdef USE_ASSERT_CHECKING + /* verify the results look sane */ + i = -1; + while ((i = bms_next_member(edis, i)) >= 0) + { + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, + i); + + Assert(bms_is_subset(relids, rinfo->clause_relids)); + } +#endif + + return edis; +} + +#ifdef USE_ASSERT_CHECKING +/* + * verify_eclass_indexes + * Verify that there are no missing references between RestrictInfos and + * EquivalenceMember's indexes, namely eclass_source_indexes and + * eclass_derive_indexes. If you modify these indexes, you should check + * them with this function. + */ +void +verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) +{ + ListCell *lc; + + /* + * All RestrictInfos in root->eq_sources must have references to + * eclass_source_indexes. + */ + foreach(lc, root->eq_sources) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_sources, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + } + } + + /* + * All RestrictInfos in root->eq_derives must have references to + * eclass_derive_indexes. + */ + foreach(lc, root->eq_derives) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + int index; + int k; + + /* Deleted members are marked NULL */ + if (rinfo == NULL) + continue; + + index = list_cell_number(root->eq_derives, lc); + k = -1; + while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) + { + /* must have a reference */ + Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + } + } +} +#endif diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index 5bc16c4bfc7..4449b8a22ee 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -36,9 +36,10 @@ static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); static void remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo); -static void remove_rel_from_restrictinfo(RestrictInfo *rinfo, +static void remove_rel_from_restrictinfo(PlannerInfo *root, + RestrictInfo *rinfo, int relid, int ojrelid); -static void remove_rel_from_eclass(EquivalenceClass *ec, +static void remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, int ojrelid); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static bool rel_supports_distinctness(PlannerInfo *root, RelOptInfo *rel); @@ -445,7 +446,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * that any such PHV is safe (and updated its ph_eval_at), so we * can just drop those references. */ - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); /* * Cross-check that the clause itself does not reference the @@ -474,7 +475,7 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) if (bms_is_member(relid, ec->ec_relids) || bms_is_member(ojrelid, ec->ec_relids)) - remove_rel_from_eclass(ec, relid, ojrelid); + remove_rel_from_eclass(root, ec, relid, ojrelid); } /* @@ -547,19 +548,28 @@ remove_rel_from_query(PlannerInfo *root, int relid, SpecialJoinInfo *sjinfo) * we have to also clean up the sub-clauses. */ static void -remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) +remove_rel_from_restrictinfo(PlannerInfo *root, RestrictInfo *rinfo, int relid, + int ojrelid) { + Relids new_clause_relids; + /* * initsplan.c is fairly cavalier about allowing RestrictInfos to share * relid sets with other RestrictInfos, and SpecialJoinInfos too. Make * sure this RestrictInfo has its own relid sets before we modify them. - * (In present usage, clause_relids is probably not shared, but - * required_relids could be; let's not assume anything.) + * The 'new_clause_relids' passed to update_clause_relids() must be a + * different instance from the current rinfo->clause_relids, so we make a + * copy of it. + */ + new_clause_relids = bms_copy(rinfo->clause_relids); + new_clause_relids = bms_del_member(new_clause_relids, relid); + new_clause_relids = bms_del_member(new_clause_relids, ojrelid); + update_clause_relids(root, rinfo, new_clause_relids); + + /* + * In present usage, required_relids could be shared, so we make a copy of + * it. */ - rinfo->clause_relids = bms_copy(rinfo->clause_relids); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, relid); - rinfo->clause_relids = bms_del_member(rinfo->clause_relids, ojrelid); - /* Likewise for required_relids */ rinfo->required_relids = bms_copy(rinfo->required_relids); rinfo->required_relids = bms_del_member(rinfo->required_relids, relid); rinfo->required_relids = bms_del_member(rinfo->required_relids, ojrelid); @@ -584,14 +594,14 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) { RestrictInfo *rinfo2 = lfirst_node(RestrictInfo, lc2); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } else { RestrictInfo *rinfo2 = castNode(RestrictInfo, orarg); - remove_rel_from_restrictinfo(rinfo2, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo2, relid, ojrelid); } } } @@ -607,9 +617,11 @@ remove_rel_from_restrictinfo(RestrictInfo *rinfo, int relid, int ojrelid) * level(s). */ static void -remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) +remove_rel_from_eclass(PlannerInfo *root, EquivalenceClass *ec, int relid, + int ojrelid) { ListCell *lc; + int i; /* Fix up the EC's overall relids */ ec->ec_relids = bms_del_member(ec->ec_relids, relid); @@ -636,11 +648,12 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) } /* Fix up the source clauses, in case we can re-use them later */ - foreach(lc, ec->ec_sources) + i = -1; + while ((i = bms_next_member(ec->ec_source_indexes, i)) >= 0) { - RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); + RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, i); - remove_rel_from_restrictinfo(rinfo, relid, ojrelid); + remove_rel_from_restrictinfo(root, rinfo, relid, ojrelid); } /* @@ -648,7 +661,7 @@ remove_rel_from_eclass(EquivalenceClass *ec, int relid, int ojrelid) * drop them. (At this point, any such clauses would be base restriction * clauses, which we'd not need anymore anyway.) */ - ec->ec_derives = NIL; + ec->ec_derive_indexes = NULL; } /* diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index b665a7762ec..ca8af0bd4b2 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -663,6 +663,8 @@ subquery_planner(PlannerGlobal *glob, Query *parse, PlannerInfo *parent_root, root->multiexpr_params = NIL; root->join_domains = NIL; root->eq_classes = NIL; + root->eq_sources = NIL; + root->eq_derives = NIL; root->ec_merging_done = false; root->last_rinfo_serial = 0; root->all_result_relids = diff --git a/src/backend/optimizer/prep/prepjointree.c b/src/backend/optimizer/prep/prepjointree.c index 2ebd938f6bd..b86e0816d70 100644 --- a/src/backend/optimizer/prep/prepjointree.c +++ b/src/backend/optimizer/prep/prepjointree.c @@ -1153,6 +1153,8 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte, subroot->multiexpr_params = NIL; subroot->join_domains = NIL; subroot->eq_classes = NIL; + subroot->eq_sources = NIL; + subroot->eq_derives = NIL; subroot->ec_merging_done = false; subroot->last_rinfo_serial = 0; subroot->all_result_relids = NULL; diff --git a/src/backend/optimizer/util/appendinfo.c b/src/backend/optimizer/util/appendinfo.c index 45e8b74f944..db88047f8b7 100644 --- a/src/backend/optimizer/util/appendinfo.c +++ b/src/backend/optimizer/util/appendinfo.c @@ -495,6 +495,8 @@ adjust_appendrel_attrs_mutator(Node *node, newinfo->right_bucketsize = -1; newinfo->left_mcvfreq = -1; newinfo->right_mcvfreq = -1; + newinfo->eq_sources_index = -1; + newinfo->eq_derives_index = -1; return (Node *) newinfo; } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5bd..b5d3984219e 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,6 +492,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); + /* + * We do not want to inherit the EquivalenceMember indexes of the parent + * to its child + */ + childrte->eclass_source_indexes = NULL; + childrte->eclass_derive_indexes = NULL; + Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/restrictinfo.c b/src/backend/optimizer/util/restrictinfo.c index 9e1458401c2..e15ef139dec 100644 --- a/src/backend/optimizer/util/restrictinfo.c +++ b/src/backend/optimizer/util/restrictinfo.c @@ -238,6 +238,9 @@ make_plain_restrictinfo(PlannerInfo *root, restrictinfo->left_hasheqoperator = InvalidOid; restrictinfo->right_hasheqoperator = InvalidOid; + restrictinfo->eq_sources_index = -1; + restrictinfo->eq_derives_index = -1; + return restrictinfo; } @@ -394,6 +397,8 @@ commute_restrictinfo(RestrictInfo *rinfo, Oid comm_op) result->right_mcvfreq = rinfo->left_mcvfreq; result->left_hasheqoperator = InvalidOid; result->right_hasheqoperator = InvalidOid; + result->eq_sources_index = -1; + result->eq_derives_index = -1; return result; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 0f9462493e3..d28e2034d09 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1247,6 +1247,12 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); + Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources + * list for RestrictInfos that mention + * this relation */ + Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives + * list for RestrictInfos that mention + * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index a8cc7b19ff2..313d907af94 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -321,6 +321,12 @@ struct PlannerInfo /* list of active EquivalenceClasses */ List *eq_classes; + /* list of source RestrictInfos used to build EquivalenceClasses */ + List *eq_sources; + + /* list of RestrictInfos derived from EquivalenceClasses */ + List *eq_derives; + /* set true once ECs are canonical */ bool ec_merging_done; @@ -1399,6 +1405,24 @@ typedef struct JoinDomain * its comment for usage. The approach to lookup child members quickly is * described as iterate_child_rel_equivalences() comment. * + * At various locations in the query planner, we must search for source and + * derived RestrictInfos regarding a given EquivalenceClass. For the common + * case, an EquivalenceClass does not have a large number of RestrictInfos, + * however, in cases such as planning queries to partitioned tables, the + * number of members can become large. To maintain planning performance, we + * make use of a bitmap index to allow us to quickly find RestrictInfos in a + * given EquivalenceClass belonging to a given relation or set of relations. + * This is done by storing a list of RestrictInfos belonging to all + * EquivalenceClasses in PlannerInfo and storing a Bitmapset for each + * RelOptInfo which has a bit set for each RestrictInfo in that list which + * relates to the given relation. We also store a Bitmapset to mark all of + * the indexes in the PlannerInfo's list of RestrictInfos in the + * EquivalenceClass. We can quickly find the interesting indexes into the + * PlannerInfo's list by performing a bit-wise AND on the RelOptInfo's + * Bitmapset and the EquivalenceClasses. RestrictInfos must be looked up in + * PlannerInfo by this technique using the ec_source_indexes and + * ec_derive_indexes Bitmapsets. + * * NB: if ec_merged isn't NULL, this class has been merged into another, and * should be ignored in favor of using the pointed-to class. * @@ -1417,8 +1441,10 @@ typedef struct EquivalenceClass List *ec_opfamilies; /* btree operator family OIDs */ Oid ec_collation; /* collation, if datatypes are collatable */ List *ec_members; /* list of EquivalenceMembers */ - List *ec_sources; /* list of generating RestrictInfos */ - List *ec_derives; /* list of derived RestrictInfos */ + Bitmapset *ec_source_indexes; /* indexes into PlannerInfo's eq_sources + * list of generating RestrictInfos */ + Bitmapset *ec_derive_indexes; /* indexes into PlannerInfo's eq_derives + * list of derived RestrictInfos */ Relids ec_relids; /* all relids appearing in ec_members, except * for child members (see below) */ bool ec_has_const; /* any pseudoconstants in ec_members? */ @@ -2659,7 +2685,12 @@ typedef struct RestrictInfo /* number of base rels in clause_relids */ int num_base_rels pg_node_attr(equal_ignore); - /* The relids (varnos+varnullingrels) actually referenced in the clause: */ + /* + * The relids (varnos+varnullingrels) actually referenced in the clause. + * + * NOTE: This field must be updated through update_clause_relids(), not by + * setting the field directly. + */ Relids clause_relids pg_node_attr(equal_ignore); /* The set of relids required to evaluate the clause: */ @@ -2777,6 +2808,10 @@ typedef struct RestrictInfo /* hash equality operators used for memoize nodes, else InvalidOid */ Oid left_hasheqoperator pg_node_attr(equal_ignore); Oid right_hasheqoperator pg_node_attr(equal_ignore); + + /* the index within root->eq_sources and root->eq_derives */ + int eq_sources_index pg_node_attr(equal_ignore); + int eq_derives_index pg_node_attr(equal_ignore); } RestrictInfo; /* diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 50812e3a5d8..a24a5801c55 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -129,6 +129,8 @@ extern Expr *canonicalize_ec_expression(Expr *expr, Oid req_type, Oid req_collation); extern void reconsider_outer_join_clauses(PlannerInfo *root); extern void rebuild_eclass_attr_needed(PlannerInfo *root); +extern void update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, + Relids new_clause_relids); extern EquivalenceClass *get_eclass_for_sort_expr(PlannerInfo *root, Expr *expr, List *opfamilies, @@ -165,7 +167,8 @@ extern bool exprs_known_equal(PlannerInfo *root, Node *item1, Node *item2, extern EquivalenceClass *match_eclasses_to_foreign_key_col(PlannerInfo *root, ForeignKeyOptInfo *fkinfo, int colno); -extern RestrictInfo *find_derived_clause_for_ec_member(EquivalenceClass *ec, +extern RestrictInfo *find_derived_clause_for_ec_member(PlannerInfo *root, + EquivalenceClass *ec, EquivalenceMember *em); extern void add_child_rel_equivalences(PlannerInfo *root, AppendRelInfo *appinfo, @@ -204,6 +207,22 @@ extern bool eclass_useful_for_merging(PlannerInfo *root, extern bool is_redundant_derived_clause(RestrictInfo *rinfo, List *clauselist); extern bool is_redundant_with_indexclauses(RestrictInfo *rinfo, List *indexclauses); +extern Bitmapset *get_ec_source_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_source_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +extern Bitmapset *get_ec_derive_indexes_strict(PlannerInfo *root, + EquivalenceClass *ec, + Relids relids); +#ifdef USE_ASSERT_CHECKING +extern void verify_eclass_indexes(PlannerInfo *root, + EquivalenceClass *ec); +#endif /* * pathkeys.c -- 2.45.2.windows.1 [application/octet-stream] v28-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch (14.4K, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/5-v28-0003-Move-EquivalenceClass-indexes-to-PlannerInfo.patch) download | inline diff: From dcb64602b2b3d33f466d15d30b65f379264833d3 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 19 Jan 2024 11:43:29 +0900 Subject: [PATCH v28 3/7] Move EquivalenceClass indexes to PlannerInfo In the previous commit, the indexes, namely ec_[source|derive]_indexes, were in RestrictInfo. This was a workaround because RelOptInfo was the best place but some RelOptInfos can be NULL and we cannot store indexes for them. This commit introduces a new struct, EquivalenceClassIndexes. This struct exists in PlannerInfo and holds our indexes. This change prevents a serialization problem with RestirctInfo in the previous commit. --- src/backend/nodes/outfuncs.c | 2 - src/backend/nodes/readfuncs.c | 2 - src/backend/optimizer/path/equivclass.c | 83 ++++++++++++------------- src/backend/optimizer/util/inherit.c | 7 --- src/backend/optimizer/util/relnode.c | 6 ++ src/include/nodes/parsenodes.h | 6 -- src/include/nodes/pathnodes.h | 26 ++++++++ src/tools/pgindent/typedefs.list | 1 + 8 files changed, 74 insertions(+), 59 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index bd6a79b57a7..8a635a2a12b 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -573,8 +573,6 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node) WRITE_BOOL_FIELD(lateral); WRITE_BOOL_FIELD(inFromCl); WRITE_NODE_FIELD(securityQuals); - WRITE_BITMAPSET_FIELD(eclass_source_indexes); - WRITE_BITMAPSET_FIELD(eclass_derive_indexes); } static void diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c index 70864fc1b29..be5f19dd7f6 100644 --- a/src/backend/nodes/readfuncs.c +++ b/src/backend/nodes/readfuncs.c @@ -434,8 +434,6 @@ _readRangeTblEntry(void) READ_BOOL_FIELD(lateral); READ_BOOL_FIELD(inFromCl); READ_NODE_FIELD(securityQuals); - READ_BITMAPSET_FIELD(eclass_source_indexes); - READ_BITMAPSET_FIELD(eclass_derive_indexes); READ_DONE(); } diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 474231499e6..96869ea80e8 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -613,10 +613,10 @@ add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_source_indexes = bms_add_member(rte->eclass_source_indexes, - source_idx); + index->source_indexes = bms_add_member(index->source_indexes, + source_idx); } } @@ -637,10 +637,10 @@ add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo) i = -1; while ((i = bms_next_member(rinfo->clause_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rte->eclass_derive_indexes = bms_add_member(rte->eclass_derive_indexes, - derive_idx); + index->derive_indexes = bms_add_member(index->derive_indexes, + derive_idx); } } @@ -679,22 +679,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(removing_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_del_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_del_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_del_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_del_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -708,22 +708,22 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(adding_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) { Assert(!bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); - rte->eclass_source_indexes = - bms_add_member(rte->eclass_source_indexes, + index->source_indexes)); + index->source_indexes = + bms_add_member(index->source_indexes, rinfo->eq_sources_index); } if (rinfo->eq_derives_index != -1) { Assert(!bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); - rte->eclass_derive_indexes = - bms_add_member(rte->eclass_derive_indexes, + index->derive_indexes)); + index->derive_indexes = + bms_add_member(index->derive_indexes, rinfo->eq_derives_index); } } @@ -738,14 +738,14 @@ update_clause_relids(PlannerInfo *root, RestrictInfo *rinfo, i = -1; while ((i = bms_next_member(common_relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; if (rinfo->eq_sources_index != -1) Assert(bms_is_member(rinfo->eq_sources_index, - rte->eclass_source_indexes)); + index->source_indexes)); if (rinfo->eq_derives_index != -1) Assert(bms_is_member(rinfo->eq_derives_index, - rte->eclass_derive_indexes)); + index->derive_indexes)); } bms_free(common_relids); #endif @@ -3970,9 +3970,9 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_esis = bms_add_members(rel_esis, rte->eclass_source_indexes); + rel_esis = bms_add_members(rel_esis, index->source_indexes); } #ifdef USE_ASSERT_CHECKING @@ -4008,7 +4008,7 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -4017,12 +4017,12 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ esis = bms_intersect(ec->ec_source_indexes, - rte->eclass_source_indexes); + index->source_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - esis = bms_int_members(esis, rte->eclass_source_indexes); + index = &root->eclass_indexes_array[i]; + esis = bms_int_members(esis, index->source_indexes); } } @@ -4059,9 +4059,9 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) while ((i = bms_next_member(relids, i)) >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; - rel_edis = bms_add_members(rel_edis, rte->eclass_derive_indexes); + rel_edis = bms_add_members(rel_edis, index->derive_indexes); } #ifdef USE_ASSERT_CHECKING @@ -4097,7 +4097,7 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, if (i >= 0) { - RangeTblEntry *rte = root->simple_rte_array[i]; + EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting @@ -4106,12 +4106,12 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, - rte->eclass_derive_indexes); + index->derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { - rte = root->simple_rte_array[i]; - edis = bms_int_members(edis, rte->eclass_derive_indexes); + index = &root->eclass_indexes_array[i]; + edis = bms_int_members(edis, index->derive_indexes); } } @@ -4134,9 +4134,8 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, /* * verify_eclass_indexes * Verify that there are no missing references between RestrictInfos and - * EquivalenceMember's indexes, namely eclass_source_indexes and - * eclass_derive_indexes. If you modify these indexes, you should check - * them with this function. + * EquivalenceMember's indexes, namely source_indexes and derive_indexes. + * If you modify these indexes, you should check them with this function. */ void verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) @@ -4145,7 +4144,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) /* * All RestrictInfos in root->eq_sources must have references to - * eclass_source_indexes. + * source_indexes. */ foreach(lc, root->eq_sources) { @@ -4162,13 +4161,13 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_source_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].source_indexes)); } } /* * All RestrictInfos in root->eq_derives must have references to - * eclass_derive_indexes. + * derive_indexes. */ foreach(lc, root->eq_derives) { @@ -4185,7 +4184,7 @@ verify_eclass_indexes(PlannerInfo *root, EquivalenceClass *ec) while ((k = bms_next_member(rinfo->clause_relids, k)) >= 0) { /* must have a reference */ - Assert(bms_is_member(index, root->simple_rte_array[k]->eclass_derive_indexes)); + Assert(bms_is_member(index, root->eclass_indexes_array[k].derive_indexes)); } } } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index b5d3984219e..3c2198ea5bd 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -492,13 +492,6 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, */ childrte = makeNode(RangeTblEntry); memcpy(childrte, parentrte, sizeof(RangeTblEntry)); - /* - * We do not want to inherit the EquivalenceMember indexes of the parent - * to its child - */ - childrte->eclass_source_indexes = NULL; - childrte->eclass_derive_indexes = NULL; - Assert(parentrte->rtekind == RTE_RELATION); /* else this is dubious */ childrte->relid = childOID; childrte->relkind = childrel->rd_rel->relkind; diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index ce494364173..ca1d7e91bff 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -119,6 +119,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->simple_rte_array[rti++] = rte; } + root->eclass_indexes_array = (EquivalenceClassIndexes *) + palloc0(size * sizeof(EquivalenceClassIndexes)); + /* append_rel_array is not needed if there are no AppendRelInfos */ if (root->append_rel_list == NIL) { @@ -218,6 +221,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->top_parent_relid_array = NULL; } + root->eclass_indexes_array = + repalloc0_array(root->eclass_indexes_array, EquivalenceClassIndexes, root->simple_rel_array_size, new_size); + root->simple_rel_array_size = new_size; } diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index d28e2034d09..0f9462493e3 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -1247,12 +1247,6 @@ typedef struct RangeTblEntry bool inFromCl pg_node_attr(query_jumble_ignore); /* security barrier quals to apply, if any */ List *securityQuals pg_node_attr(query_jumble_ignore); - Bitmapset *eclass_source_indexes; /* Indexes in PlannerInfo's eq_sources - * list for RestrictInfos that mention - * this relation */ - Bitmapset *eclass_derive_indexes; /* Indexes in PlannerInfo's eq_derives - * list for RestrictInfos that mention - * this relation */ } RangeTblEntry; /* diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 313d907af94..c068dc1b6c6 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -192,6 +192,8 @@ typedef struct PlannerInfo PlannerInfo; #define HAVE_PLANNERINFO_TYPEDEF 1 #endif +struct EquivalenceClassIndexes; + struct PlannerInfo { pg_node_attr(no_copy_equal, no_read, no_query_jumble) @@ -248,6 +250,13 @@ struct PlannerInfo */ struct AppendRelInfo **append_rel_array pg_node_attr(read_write_ignore); + /* + * eclass_indexes_array is the same length as simple_rel_array and holds + * the indexes of the corresponding rels for faster lookups of + * RestrictInfo. See the EquivalenceClass comment for more details. + */ + struct EquivalenceClassIndexes *eclass_indexes_array pg_node_attr(read_write_ignore); + /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within @@ -1537,6 +1546,23 @@ typedef struct List *ec_members; /* parent and child members */ } EquivalenceChildMemberIterator; +/* + * EquivalenceClassIndexes + * + * As mentioned in the EquivalenceClass comment, we introduce a + * bitmapset-based indexing mechanism for faster lookups of RestrictInfo. This + * struct exists for each relation and holds the corresponding indexes. + */ +typedef struct EquivalenceClassIndexes +{ + Bitmapset *source_indexes; /* Indexes in PlannerInfo's eq_sources list + * for RestrictInfos that mention this + * relation */ + Bitmapset *derive_indexes; /* Indexes in PlannerInfo's eq_derives list + * for RestrictInfos that mention this + * relation */ +} EquivalenceClassIndexes; + /* * PathKeys * diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 32e4be0f47f..1d9e73b8386 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -685,6 +685,7 @@ EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData EquivalenceChildMemberIterator EquivalenceClass +EquivalenceClassIndexes EquivalenceMember ErrorContextCallback ErrorData -- 2.45.2.windows.1 [application/octet-stream] v28-0004-Rename-add_eq_member-to-add_parent_eq_member.patch (5.2K, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/6-v28-0004-Rename-add_eq_member-to-add_parent_eq_member.patch) download | inline diff: From a4481ad3f3d4b41c80b39721ebe1b96e2946e405 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Wed, 21 Aug 2024 13:54:59 +0900 Subject: [PATCH v28 4/7] Rename add_eq_member() to add_parent_eq_member() After our changes, add_eq_member() no longer creates and adds child EquivalenceMembers. This commit renames it to add_parent_eq_member() to clarify that it only creates parent members, and that we need to use make_eq_member() to handle child EquivalenceMembers. --- src/backend/optimizer/path/equivclass.c | 54 ++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 96869ea80e8..e30d931f19c 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -38,10 +38,10 @@ static EquivalenceMember *make_eq_member(EquivalenceClass *ec, JoinDomain *jdomain, EquivalenceMember *parent, Oid datatype); -static EquivalenceMember *add_eq_member(EquivalenceClass *ec, - Expr *expr, Relids relids, - JoinDomain *jdomain, - Oid datatype); +static EquivalenceMember *add_parent_eq_member(EquivalenceClass *ec, + Expr *expr, Relids relids, + JoinDomain *jdomain, + Oid datatype); static void add_eq_source(PlannerInfo *root, EquivalenceClass *ec, RestrictInfo *rinfo); static void add_eq_derive(PlannerInfo *root, EquivalenceClass *ec, @@ -389,8 +389,8 @@ process_equivalence(PlannerInfo *root, else if (ec1) { /* Case 3: add item2 to ec1 */ - em2 = add_eq_member(ec1, item2, item2_relids, - jdomain, item2_type); + em2 = add_parent_eq_member(ec1, item2, item2_relids, + jdomain, item2_type); ec1->ec_min_security = Min(ec1->ec_min_security, restrictinfo->security_level); ec1->ec_max_security = Max(ec1->ec_max_security, @@ -407,8 +407,8 @@ process_equivalence(PlannerInfo *root, else if (ec2) { /* Case 3: add item1 to ec2 */ - em1 = add_eq_member(ec2, item1, item1_relids, - jdomain, item1_type); + em1 = add_parent_eq_member(ec2, item1, item1_relids, + jdomain, item1_type); ec2->ec_min_security = Min(ec2->ec_min_security, restrictinfo->security_level); ec2->ec_max_security = Max(ec2->ec_max_security, @@ -440,10 +440,10 @@ process_equivalence(PlannerInfo *root, ec->ec_min_security = restrictinfo->security_level; ec->ec_max_security = restrictinfo->security_level; ec->ec_merged = NULL; - em1 = add_eq_member(ec, item1, item1_relids, - jdomain, item1_type); - em2 = add_eq_member(ec, item2, item2_relids, - jdomain, item2_type); + em1 = add_parent_eq_member(ec, item1, item1_relids, + jdomain, item1_type); + em2 = add_parent_eq_member(ec, item2, item2_relids, + jdomain, item2_type); root->eq_classes = lappend(root->eq_classes, ec); @@ -578,7 +578,7 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, } /* - * add_eq_member - build a new parent EquivalenceMember and add it to an EC + * add_parent_eq_member - build a new parent EquivalenceMember and add it to an EC * * Note: We don't have a function to add a child member like * add_child_eq_member() because how to do it depends on the relations they are @@ -586,8 +586,8 @@ make_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, * add_child_join_rel_equivalences() to see how to add child members. */ static EquivalenceMember * -add_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, - JoinDomain *jdomain, Oid datatype) +add_parent_eq_member(EquivalenceClass *ec, Expr *expr, Relids relids, + JoinDomain *jdomain, Oid datatype) { EquivalenceMember *em = make_eq_member(ec, expr, relids, jdomain, NULL, datatype); @@ -921,14 +921,14 @@ get_eclass_for_sort_expr(PlannerInfo *root, */ expr_relids = pull_varnos(root, (Node *) expr); - newem = add_eq_member(newec, copyObject(expr), expr_relids, - jdomain, opcintype); + newem = add_parent_eq_member(newec, copyObject(expr), expr_relids, + jdomain, opcintype); /* - * add_eq_member doesn't check for volatile functions, set-returning - * functions, aggregates, or window functions, but such could appear in - * sort expressions; so we have to check whether its const-marking was - * correct. + * add_parent_eq_member doesn't check for volatile functions, + * set-returning functions, aggregates, or window functions, but such + * could appear in sort expressions; so we have to check whether its + * const-marking was correct. */ if (newec->ec_has_const) { @@ -3305,12 +3305,12 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + add_parent_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); lc2 = lnext(setop_pathkeys, lc2); } -- 2.45.2.windows.1 [application/octet-stream] v28-0005-Resolve-conflict-with-commit-66c0185.patch (6.4K, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/7-v28-0005-Resolve-conflict-with-commit-66c0185.patch) download | inline diff: From 413fa3487560a3f15bfdf0062169856cb0892fb2 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Tue, 27 Aug 2024 13:20:29 +0900 Subject: [PATCH v28 5/7] Resolve conflict with commit 66c0185 This commit resolves a conflict with 66c0185, which added add_setop_child_rel_equivalences(). The function creates child EquivalenceMembers to efficiently implement UNION queries. This commit adjusts our optimization to handle this type of child EquivalenceMembers based on UNION parent-child relationships. --- src/backend/optimizer/path/equivclass.c | 51 ++++++++++++++++++++++--- src/backend/optimizer/util/inherit.c | 8 +++- src/backend/optimizer/util/relnode.c | 10 +++-- src/include/nodes/pathnodes.h | 2 +- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index e30d931f19c..022f20020af 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -3288,7 +3288,9 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, { TargetEntry *tle = lfirst_node(TargetEntry, lc); EquivalenceMember *parent_em; + EquivalenceMember *child_em; PathKey *pk; + Index parent_relid; if (tle->resjunk) continue; @@ -3305,12 +3307,49 @@ add_setop_child_rel_equivalences(PlannerInfo *root, RelOptInfo *child_rel, * likewise, the JoinDomain can be that of the initial member of the * Pathkey's EquivalenceClass. */ - add_parent_eq_member(pk->pk_eclass, - tle->expr, - child_rel->relids, - parent_em->em_jdomain, - parent_em, - exprType((Node *) tle->expr)); + child_em = make_eq_member(pk->pk_eclass, + tle->expr, + child_rel->relids, + parent_em->em_jdomain, + parent_em, + exprType((Node *) tle->expr)); + child_rel->eclass_child_members = + lappend(child_rel->eclass_child_members, child_em); + + /* + * We save the knowledge that 'child_em' can be translated using + * 'child_rel'. This knowledge is useful for + * add_transformed_child_version() to find child members from the + * given Relids. + */ + parent_em->em_child_relids = + bms_add_member(parent_em->em_child_relids, child_rel->relid); + + /* + * Make an UNION parent-child relationship between parent_em and + * child_rel->relid. We record this relationship in + * root->top_parent_relid_array, which generally has AppendRelInfo + * relationships. We use the same array here to retrieve UNION child + * members. + * + * XXX Here we treat the first member of parent_em->em_relids as a + * parent of child_rel. Is this correct? What happens if + * parent_em->em_relids has two or more members? + */ + parent_relid = bms_next_member(parent_em->em_relids, -1); + if (root->top_parent_relid_array == NULL) + { + /* + * If the array is NULL, allocate it here. + */ + root->top_parent_relid_array = (Index *) + palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[child_rel->relid] == -1 || + root->top_parent_relid_array[child_rel->relid] == parent_relid); + root->top_parent_relid_array[child_rel->relid] = parent_relid; lc2 = lnext(setop_pathkeys, lc2); } diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 3c2198ea5bd..7b2390687e0 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -582,9 +582,13 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte, * Find a top parent rel's index and save it to top_parent_relid_array. */ if (root->top_parent_relid_array == NULL) + { root->top_parent_relid_array = - (Index *) palloc0(root->simple_rel_array_size * sizeof(Index)); - Assert(root->top_parent_relid_array[childRTindex] == 0); + (Index *) palloc(root->simple_rel_array_size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * root->simple_rel_array_size); + } + Assert(root->top_parent_relid_array[childRTindex] == -1); topParentRTindex = parentRTindex; while (root->append_rel_array[topParentRTindex] != NULL && root->append_rel_array[topParentRTindex]->parent_relid != 0) diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index ca1d7e91bff..fcd57c1f3e9 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -133,7 +133,9 @@ setup_simple_rel_arrays(PlannerInfo *root) root->append_rel_array = (AppendRelInfo **) palloc0(size * sizeof(AppendRelInfo *)); root->top_parent_relid_array = (Index *) - palloc0(size * sizeof(Index)); + palloc(size * sizeof(Index)); + MemSet(root->top_parent_relid_array, -1, + sizeof(Index) * size); /* * append_rel_array is filled with any already-existing AppendRelInfos, @@ -206,7 +208,9 @@ expand_planner_arrays(PlannerInfo *root, int add_size) root->append_rel_array = repalloc0_array(root->append_rel_array, AppendRelInfo *, root->simple_rel_array_size, new_size); root->top_parent_relid_array = - repalloc0_array(root->top_parent_relid_array, Index, root->simple_rel_array_size, new_size); + repalloc_array(root->top_parent_relid_array, Index, new_size); + MemSet(root->top_parent_relid_array + root->simple_rel_array_size, -1, + sizeof(Index) * add_size); } else { @@ -1608,7 +1612,7 @@ find_relids_top_parents_slow(PlannerInfo *root, Relids relids) { int top_parent_relid = (int) top_parent_relid_array[i]; - if (top_parent_relid == 0) + if (top_parent_relid == -1) top_parent_relid = i; /* 'i' has no parents, so add itself */ else if (top_parent_relid != i) is_top_parent = false; diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index c068dc1b6c6..9c2d284c079 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -260,7 +260,7 @@ struct PlannerInfo /* * top_parent_relid_array is the same length as simple_rel_array and holds * the top-level parent indexes of the corresponding rels within - * simple_rel_array. The element can be zero if the rel has no parents, + * simple_rel_array. The element can be -1 if the rel has no parents, * i.e., is itself in a top-level. */ Index *top_parent_relid_array pg_node_attr(read_write_ignore); -- 2.45.2.windows.1 [application/octet-stream] v28-0006-Don-t-use-likely-in-find_relids_top_parents.patch (955B, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/8-v28-0006-Don-t-use-likely-in-find_relids_top_parents.patch) download | inline diff: From 862963ef73ee580aa84951e7b515a31d6708dcf4 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 29 Nov 2024 14:49:42 +0900 Subject: [PATCH v28 6/7] Don't use likely in find_relids_top_parents --- src/include/optimizer/pathnode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 5e79cf1f63b..cf21ba29fe9 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -346,7 +346,7 @@ extern Relids find_childrel_parents(PlannerInfo *root, RelOptInfo *rel); * root->top_parent_relid_array is NULL. */ #define find_relids_top_parents(root, relids) \ - (likely((root)->top_parent_relid_array == NULL) \ + ((root)->top_parent_relid_array == NULL \ ? NULL : find_relids_top_parents_slow(root, relids)) extern Relids find_relids_top_parents_slow(PlannerInfo *root, Relids relids); -- 2.45.2.windows.1 [application/octet-stream] v28-0007-Don-t-use-unlikely-top_parent-NULL.patch (4.5K, ../../CAJ2pMkZMKwdjYRUGA0=za4SUrgpBJ5_JDYuVeG=Esbv1dKSouQ@mail.gmail.com/9-v28-0007-Don-t-use-unlikely-top_parent-NULL.patch) download | inline diff: From 68c14b66f15f544223cca11f51869e79cfb022d1 Mon Sep 17 00:00:00 2001 From: Yuya Watari <[email protected]> Date: Fri, 29 Nov 2024 14:49:43 +0900 Subject: [PATCH v28 7/7] Don't use unlikely(top_parent != NULL) --- contrib/postgres_fdw/postgres_fdw.c | 2 +- src/backend/optimizer/path/equivclass.c | 10 +++++----- src/backend/optimizer/path/indxpath.c | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index e68c116164c..030fe741030 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -7854,7 +7854,7 @@ find_em_for_rel(PlannerInfo *root, EquivalenceClass *ec, RelOptInfo *rel) * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ - if (unlikely(top_parent_rel_relids != NULL) && !em->em_is_child && + if (top_parent_rel_relids != NULL && !em->em_is_child && bms_is_subset(em->em_relids, top_parent_rel_relids)) iterate_child_rel_equivalences(&it, root, ec, em, rel->relids); diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 022f20020af..86057538c5c 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -861,7 +861,7 @@ get_eclass_for_sort_expr(PlannerInfo *root, * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ - if (unlikely(top_parent_rel != NULL) && !cur_em->em_is_child && + if (top_parent_rel != NULL && !cur_em->em_is_child && bms_equal(cur_em->em_relids, top_parent_rel)) iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel); @@ -1034,7 +1034,7 @@ find_ec_member_matching_expr(PlannerInfo *root, EquivalenceClass *ec, * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ - if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + if (top_parent_relids != NULL && !em->em_is_child && bms_is_subset(em->em_relids, top_parent_relids)) iterate_child_rel_equivalences(&it, root, ec, em, relids); @@ -1144,7 +1144,7 @@ find_computable_ec_member(PlannerInfo *root, * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ - if (unlikely(top_parent_relids != NULL) && !em->em_is_child && + if (top_parent_relids != NULL && !em->em_is_child && bms_is_subset(em->em_relids, top_parent_relids)) iterate_child_rel_equivalences(&it, root, ec, em, relids); @@ -1881,7 +1881,7 @@ generate_join_implied_equalities_normal(PlannerInfo *root, * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ - if (unlikely(top_parent_join_relids != NULL) && !cur_em->em_is_child && + if (top_parent_join_relids != NULL && !cur_em->em_is_child && bms_is_subset(cur_em->em_relids, top_parent_join_relids)) iterate_child_rel_equivalences(&it, root, ec, cur_em, join_relids); @@ -3616,7 +3616,7 @@ generate_implied_equalities_for_column(PlannerInfo *root, * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ - if (unlikely(top_parent_rel_relids != NULL) && !cur_em->em_is_child && + if (top_parent_rel_relids != NULL && !cur_em->em_is_child && bms_equal(cur_em->em_relids, top_parent_rel_relids)) iterate_child_rel_equivalences(&it, root, cur_ec, cur_em, rel->relids); if (bms_equal(cur_em->em_relids, rel->relids) && diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 4a42752486e..9b7d01a85d8 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -3787,7 +3787,7 @@ match_pathkeys_to_index(PlannerInfo *root, IndexOptInfo *index, List *pathkeys, * If child EquivalenceMembers may match the request, we add and * iterate over them by calling iterate_child_rel_equivalences(). */ - if (unlikely(top_parent_index_relids != NULL) && !member->em_is_child && + if (top_parent_index_relids != NULL && !member->em_is_child && bms_equal(member->em_relids, top_parent_index_relids)) iterate_child_rel_equivalences(&it, root, pathkey->pk_eclass, member, index->rel->relids); -- 2.45.2.windows.1 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-12-03 04:12 Ashutosh Bapat <[email protected]> parent: Yuya Watari <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Ashutosh Bapat @ 2024-12-03 04:12 UTC (permalink / raw) To: Yuya Watari <[email protected]>; +Cc: Dmitry Dolgov <[email protected]>; PostgreSQL Developers <[email protected]>; jian he <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Alvaro Herrera <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> On Mon, Dec 2, 2024 at 2:22 PM Yuya Watari <[email protected]> wrote: > > 4. Discussion > > First of all, tables 1, 2 and the figure attached to this email show > that likely and unlikely do not have the effect I expected. Rather, > tables 3, 4, 5 and 6 imply that they can have a negative effect on > queries A and B. So it is better to remove these likely and unlikely. > > For the design change, the benchmark results show that it may cause > some regression, especially for smaller sizes. However, Figure 1 also > shows that the regression is much smaller than its variance. This > design change is intended to improve code maintainability. The > regression is small enough that I think these results are acceptable. > What do you think about this? > > [1] https://www.postgresql.org/message-id/[email protected]... > [2] https://www.postgresql.org/message-id/[email protected]... > Hi Yuya, For one of the earlier versions, I had reported a large memory consumption in all cases and increase in planning time for Assert enabled builds. How does the latest version perform in those aspects? -- Best Wishes, Ashutosh Bapat ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-12-03 10:38 Alvaro Herrera <[email protected]> parent: Ashutosh Bapat <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Alvaro Herrera @ 2024-12-03 10:38 UTC (permalink / raw) To: Ashutosh Bapat <[email protected]>; +Cc: Yuya Watari <[email protected]>; Dmitry Dolgov <[email protected]>; PostgreSQL Developers <[email protected]>; jian he <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> Hello, On 2024-Dec-03, Ashutosh Bapat wrote: > For one of the earlier versions, I had reported a large memory > consumption in all cases and increase in planning time for Assert > enabled builds. How does the latest version perform in those aspects? I don't think planning time in assert-enabled builds is something we should worry about, at all. Planning time in production builds is the important one. -- Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/ "Learn about compilers. Then everything looks like either a compiler or a database, and now you have two problems but one of them is fun." https://twitter.com/thingskatedid/status/1456027786158776329 ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-12-03 12:10 Ashutosh Bapat <[email protected]> parent: Alvaro Herrera <[email protected]> 0 siblings, 1 reply; 21+ messages in thread From: Ashutosh Bapat @ 2024-12-03 12:10 UTC (permalink / raw) To: Alvaro Herrera <[email protected]>; +Cc: Yuya Watari <[email protected]>; Dmitry Dolgov <[email protected]>; PostgreSQL Developers <[email protected]>; jian he <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> On Tue, Dec 3, 2024 at 4:08 PM Alvaro Herrera <[email protected]> wrote: > > Hello, > > On 2024-Dec-03, Ashutosh Bapat wrote: > > > For one of the earlier versions, I had reported a large memory > > consumption in all cases and increase in planning time for Assert > > enabled builds. How does the latest version perform in those aspects? > > I don't think planning time in assert-enabled builds is something we > should worry about, at all. Planning time in production builds is the > important one. > This was discussed earlier. See a few emails from [1] going backwards. The degradation was Nx, if I am reading those emails right. That means somebody who is working with a large number of partitions has to spend Nx time in running their tests. Given that the planning time with thousands of partitions is already in seconds, slowing that further down, even in an assert build is slowing development down further. My suggestion of using OPTIMIZER_DEBUG will help us keep the sanity checks and also not slow down development. [1] https://www.postgresql.org/message-id/[email protected]... -- Best Wishes, Ashutosh Bapat ^ permalink raw reply [nested|flat] 21+ messages in thread
* Re: [PoC] Reducing planning time when tables have many partitions @ 2024-12-11 03:16 Yuya Watari <[email protected]> parent: Ashutosh Bapat <[email protected]> 0 siblings, 0 replies; 21+ messages in thread From: Yuya Watari @ 2024-12-11 03:16 UTC (permalink / raw) To: Ashutosh Bapat <[email protected]>; Alvaro Herrera <[email protected]>; +Cc: Dmitry Dolgov <[email protected]>; PostgreSQL Developers <[email protected]>; jian he <[email protected]>; Alena Rybakina <[email protected]>; Andrei Lepikhov <[email protected]>; David Rowley <[email protected]>; Thom Brown <[email protected]>; Zhang Mingli <[email protected]>; Tom Lane <[email protected]> Hello Ashutosh and Alvaro, I appreciate you replying to the email. On Tue, Dec 3, 2024 at 7:38 PM Alvaro Herrera <[email protected]> wrote: > > I don't think planning time in assert-enabled builds is something we > should worry about, at all. Planning time in production builds is the > important one. Thank you for your reply. Making debug builds too slow is not good for developers, so I'd like to see how these patches behave with assert-enabled builds. On Tue, Dec 3, 2024 at 1:12 PM Ashutosh Bapat <[email protected]> wrote: > > Hi Yuya, > For one of the earlier versions, I had reported a large memory > consumption in all cases and increase in planning time for Assert > enabled builds. How does the latest version perform in those aspects? 1. Experimental results I ran experiments to measure memory consumption during planning. These are done with the release build. In the experiments, I used the rebased version of your patch [1], which is attached to this email. Table 1: Memory consumption when planning query A (without partition-wise join (PWJ), MiB) --------------------------------- n | Master | v23 | v28 --------------------------------- 1 | 0.132 | 0.137 | 0.137 2 | 0.158 | 0.166 | 0.166 4 | 0.220 | 0.234 | 0.235 8 | 0.347 | 0.375 | 0.375 16 | 0.596 | 0.652 | 0.653 32 | 1.104 | 1.221 | 1.223 64 | 2.126 | 2.392 | 2.396 128 | 4.245 | 4.917 | 4.925 256 | 8.742 | 10.651 | 10.663 384 | 13.603 | 17.159 | 17.176 512 | 18.758 | 24.827 | 24.850 640 | 23.924 | 32.223 | 32.253 768 | 30.050 | 41.843 | 41.879 896 | 36.224 | 51.937 | 51.978 1024 | 42.923 | 64.058 | 64.105 --------------------------------- Table 2: Memory consumption when planning query A (with PWJ, MiB) ------------------------------------ n | Master | v23 | v28 ------------------------------------ 1 | 0.190 | 0.194 | 0.195 2 | 0.276 | 0.284 | 0.284 4 | 0.461 | 0.475 | 0.475 8 | 0.844 | 0.871 | 0.871 16 | 1.584 | 1.640 | 1.641 32 | 3.085 | 3.202 | 3.204 64 | 6.099 | 6.365 | 6.369 128 | 12.261 | 12.934 | 12.941 256 | 25.061 | 26.970 | 26.982 384 | 38.542 | 42.098 | 42.116 512 | 52.541 | 58.610 | 58.633 640 | 66.579 | 74.878 | 74.908 768 | 82.421 | 94.214 | 94.250 896 | 98.483 | 114.196 | 114.237 1024 | 115.074 | 136.208 | 136.255 ------------------------------------ Table 3: Memory consumption when planning query B (without PWJ, MiB) ---------------------------------- n | Master | v23 | v28 ---------------------------------- 1 | 16.019 | 16.404 | 16.404 2 | 15.288 | 15.708 | 15.708 4 | 15.674 | 16.360 | 16.360 8 | 16.554 | 17.784 | 17.786 16 | 18.221 | 19.954 | 19.958 32 | 21.630 | 25.609 | 25.617 64 | 28.913 | 39.419 | 39.427 128 | 45.331 | 77.015 | 77.030 256 | 86.127 | 192.884 | 192.916 ---------------------------------- Table 4: Memory consumption when planning query B (with PWJ, MiB) -------------------------------------- n | Master | v23 | v28 -------------------------------------- 1 | 33.623 | 34.008 | 34.008 2 | 50.285 | 50.705 | 50.705 4 | 85.562 | 86.247 | 86.247 8 | 156.465 | 157.695 | 157.697 16 | 298.692 | 300.424 | 300.428 32 | 585.713 | 589.692 | 589.699 64 | 1169.396 | 1179.901 | 1179.909 128 | 2375.592 | 2407.275 | 2407.291 256 | 4942.295 | 5049.053 | 5049.084 -------------------------------------- Next, I measured the planning times using the debug build with assertions. In this experiment, I set CFLAGS to "-O0" and also used the attached patch that removes assertions in Bitmapset-based indexes. Table 5: Planning time of query A (debug build, ms) ----------------------------------------- n | Master | v28 | v28 w/ patch ----------------------------------------- 1 | 0.648 | 0.664 | 0.665 2 | 0.788 | 0.810 | 0.800 4 | 0.891 | 0.936 | 0.931 8 | 1.202 | 1.301 | 1.268 16 | 1.973 | 2.145 | 2.042 32 | 3.668 | 4.000 | 3.638 64 | 8.093 | 8.597 | 7.167 128 | 20.015 | 19.641 | 14.274 256 | 57.634 | 51.008 | 29.930 384 | 114.280 | 94.760 | 46.449 512 | 196.492 | 154.230 | 63.758 640 | 315.037 | 240.142 | 82.476 768 | 466.149 | 338.043 | 101.318 896 | 679.029 | 511.097 | 134.854 1024 | 897.806 | 592.823 | 141.852 ----------------------------------------- Table 6: Planning time of query B (debug build, ms) ------------------------------------------ n | Master | v28 | v28 w/ patch ------------------------------------------ 1 | 43.788 | 46.364 | 45.418 2 | 42.637 | 45.750 | 44.093 4 | 43.842 | 48.109 | 45.000 8 | 47.504 | 54.410 | 48.199 16 | 55.682 | 67.242 | 53.895 32 | 77.736 | 98.507 | 66.877 64 | 144.772 | 185.697 | 96.591 128 | 411.967 | 503.644 | 166.437 256 | 1653.681 | 1610.697 | 337.940 ------------------------------------------ 2. Discussion Tables 1, 2, 3, and 4 show that the proposed patches increase memory consumption. There seems to be no difference between v23 and v28. The increase is more significant for query B, which is 2x or more. For debug builds, I observed regressions compared to the master. The regressions were reduced with the attached patch. This indicates that the get_ec_[source|derive]_indexes[_strict]() functions (quoted below) have time-consuming assertions. I think these assertions are helpful, but it might be better to remove them to avoid slowing down debug builds. What do you think? ===== Bitmapset * get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, Relids relids) { Bitmapset *edis = NULL; int i = bms_next_member(relids, -1); if (i >= 0) { EquivalenceClassIndexes *index = &root->eclass_indexes_array[i]; /* * bms_intersect to the first relation to try to keep the resulting * Bitmapset as small as possible. This saves having to make a * complete bms_copy() of one of them. One may contain significantly * more words than the other. */ edis = bms_intersect(ec->ec_derive_indexes, index->derive_indexes); while ((i = bms_next_member(relids, i)) >= 0) { index = &root->eclass_indexes_array[i]; edis = bms_int_members(edis, index->derive_indexes); } } #ifdef USE_ASSERT_CHECKING /* verify the results look sane */ i = -1; while ((i = bms_next_member(edis, i)) >= 0) { RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, i); Assert(bms_is_subset(relids, rinfo->clause_relids)); } #endif return edis; } ===== [1] https://www.postgresql.org/message-id/[email protected]... -- Best regards, Yuya Watari From 1c5bdc574b9ab136ec29a2bd86b43fb484a57345 Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat <[email protected]> Date: Wed, 12 Jul 2023 14:34:14 +0530 Subject: [PATCH] Report memory used for planning a query in EXPLAIN ANALYZE The memory used in the CurrentMemoryContext and its children is sampled before and after calling pg_plan_query() from ExplainOneQuery(). The difference in the two samples is reported as the memory consumed while planning the query. This may not account for the memory allocated in memory contexts which are not children of CurrentMemoryContext. These contexts are usually other long lived contexts, e.g. CacheMemoryContext, which are shared by all the queries run in a session. The consumption in those can not be attributed only to a given query and hence should not be reported any way. The memory consumption is reported as "Planning Memory" property in EXPLAIN ANALYZE output. Ashutosh Bapat --- src/backend/commands/explain.c | 12 ++++++++++-- src/backend/commands/prepare.c | 5 ++++- src/backend/utils/mmgr/mcxt.c | 19 +++++++++++++++++++ src/include/commands/explain.h | 3 ++- src/include/utils/memutils.h | 1 + 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index a3f1d53d7a..42fe11b3ce 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -471,6 +471,7 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; + Size mem_consumed; if (es->memory) { @@ -491,12 +492,14 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, if (es->buffers) bufusage_start = pgBufferUsage; INSTR_TIME_SET_CURRENT(planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext); /* plan the query */ plan = pg_plan_query(query, queryString, cursorOptions, params); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext) - mem_consumed; if (es->memory) { @@ -514,7 +517,7 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, /* run it (if needed) and produce output */ ExplainOnePlan(plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL), - es->memory ? &mem_counters : NULL); + es->memory ? &mem_counters : NULL, &mem_consumed); } /* @@ -638,7 +641,8 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, - const MemoryContextCounters *mem_counters) + const MemoryContextCounters *mem_counters, + const Size *mem_consumed) { DestReceiver *dest; QueryDesc *queryDesc; @@ -769,6 +773,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, double plantime = INSTR_TIME_GET_DOUBLE(*planduration); ExplainPropertyFloat("Planning Time", "ms", 1000.0 * plantime, 3, es); + + if (mem_consumed) + ExplainPropertyUInteger("Planning Memory", "bytes", + (uint64) *mem_consumed, es); } /* Print info about runtime of triggers */ diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c index a93f970a29..e5a49e825e 100644 --- a/src/backend/commands/prepare.c +++ b/src/backend/commands/prepare.c @@ -582,6 +582,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; + Size mem_consumed; if (es->memory) { @@ -596,6 +597,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, if (es->buffers) bufusage_start = pgBufferUsage; INSTR_TIME_SET_CURRENT(planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext); /* Look it up in the hash table */ entry = FetchPreparedStatement(execstmt->name, true); @@ -632,6 +634,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext) - mem_consumed; if (es->memory) { @@ -656,7 +659,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, if (pstmt->commandType != CMD_UTILITY) ExplainOnePlan(pstmt, into, es, query_string, paramLI, pstate->p_queryEnv, &planduration, (es->buffers ? &bufusage : NULL), - es->memory ? &mem_counters : NULL); + es->memory ? &mem_counters : NULL, &mem_consumed); else ExplainOneUtility(pstmt->utilityStmt, into, es, pstate, paramLI); diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index 70d33226cb..e0a119a886 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -865,6 +865,25 @@ MemoryContextStatsDetail(MemoryContext context, } } +/* + * Compute the memory used in the given context and its children. + * + * XXX: Instead of this separate function we could modify + * MemoryContextStatsDetail() to report used memory and disable printing the + * detailed stats. + */ +extern Size +MemoryContextMemUsed(MemoryContext context) +{ + MemoryContextCounters grand_totals; + + memset(&grand_totals, 0, sizeof(grand_totals)); + + MemoryContextStatsInternal(context, 0, false, 100, &grand_totals, false); + + return grand_totals.totalspace - grand_totals.freespace; +} + /* * MemoryContextStatsInternal * One recursion level for MemoryContextStats diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index aa5872bc15..4fb267afef 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -108,7 +108,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, - const MemoryContextCounters *mem_counters); + const MemoryContextCounters *mem_counters, + const Size *mem_consumed); extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc); extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc); diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h index bf93433b78..9afb0bfb2a 100644 --- a/src/include/utils/memutils.h +++ b/src/include/utils/memutils.h @@ -92,6 +92,7 @@ extern void MemoryContextStatsDetail(MemoryContext context, bool print_to_stderr); extern void MemoryContextAllowInCriticalSection(MemoryContext context, bool allow); +extern Size MemoryContextMemUsed(MemoryContext context); #ifdef MEMORY_CONTEXT_CHECKING extern void MemoryContextCheck(MemoryContext context); -- 2.45.2.windows.1 diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 86057538c5..9bbe11a9de 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -4014,18 +4014,6 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) rel_esis = bms_add_members(rel_esis, index->source_indexes); } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(rel_esis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, - i); - - Assert(bms_overlap(relids, rinfo->clause_relids)); - } -#endif - /* bitwise-AND to leave only the ones for this EquivalenceClass */ return bms_int_members(rel_esis, ec->ec_source_indexes); } @@ -4065,18 +4053,6 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, } } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(esis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, - i); - - Assert(bms_is_subset(relids, rinfo->clause_relids)); - } -#endif - return esis; } @@ -4103,18 +4079,6 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) rel_edis = bms_add_members(rel_edis, index->derive_indexes); } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(rel_edis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, - i); - - Assert(bms_overlap(relids, rinfo->clause_relids)); - } -#endif - /* bitwise-AND to leave only the ones for this EquivalenceClass */ return bms_int_members(rel_edis, ec->ec_derive_indexes); } @@ -4154,18 +4118,6 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, } } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(edis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, - i); - - Assert(bms_is_subset(relids, rinfo->clause_relids)); - } -#endif - return edis; } Attachments: [text/plain] Report-memory-used-for-planning-a-query-in-EXPLAIN-A.txt (6.9K, ../../CAJ2pMkY-Vd55REn3N1ACuRS0UT5QEe38r2pJ6=MatLKZkXHOxg@mail.gmail.com/2-Report-memory-used-for-planning-a-query-in-EXPLAIN-A.txt) download | inline diff: From 1c5bdc574b9ab136ec29a2bd86b43fb484a57345 Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat <[email protected]> Date: Wed, 12 Jul 2023 14:34:14 +0530 Subject: [PATCH] Report memory used for planning a query in EXPLAIN ANALYZE The memory used in the CurrentMemoryContext and its children is sampled before and after calling pg_plan_query() from ExplainOneQuery(). The difference in the two samples is reported as the memory consumed while planning the query. This may not account for the memory allocated in memory contexts which are not children of CurrentMemoryContext. These contexts are usually other long lived contexts, e.g. CacheMemoryContext, which are shared by all the queries run in a session. The consumption in those can not be attributed only to a given query and hence should not be reported any way. The memory consumption is reported as "Planning Memory" property in EXPLAIN ANALYZE output. Ashutosh Bapat --- src/backend/commands/explain.c | 12 ++++++++++-- src/backend/commands/prepare.c | 5 ++++- src/backend/utils/mmgr/mcxt.c | 19 +++++++++++++++++++ src/include/commands/explain.h | 3 ++- src/include/utils/memutils.h | 1 + 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index a3f1d53d7a..42fe11b3ce 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -471,6 +471,7 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; + Size mem_consumed; if (es->memory) { @@ -491,12 +492,14 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, if (es->buffers) bufusage_start = pgBufferUsage; INSTR_TIME_SET_CURRENT(planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext); /* plan the query */ plan = pg_plan_query(query, queryString, cursorOptions, params); INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext) - mem_consumed; if (es->memory) { @@ -514,7 +517,7 @@ standard_ExplainOneQuery(Query *query, int cursorOptions, /* run it (if needed) and produce output */ ExplainOnePlan(plan, into, es, queryString, params, queryEnv, &planduration, (es->buffers ? &bufusage : NULL), - es->memory ? &mem_counters : NULL); + es->memory ? &mem_counters : NULL, &mem_consumed); } /* @@ -638,7 +641,8 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, - const MemoryContextCounters *mem_counters) + const MemoryContextCounters *mem_counters, + const Size *mem_consumed) { DestReceiver *dest; QueryDesc *queryDesc; @@ -769,6 +773,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, double plantime = INSTR_TIME_GET_DOUBLE(*planduration); ExplainPropertyFloat("Planning Time", "ms", 1000.0 * plantime, 3, es); + + if (mem_consumed) + ExplainPropertyUInteger("Planning Memory", "bytes", + (uint64) *mem_consumed, es); } /* Print info about runtime of triggers */ diff --git a/src/backend/commands/prepare.c b/src/backend/commands/prepare.c index a93f970a29..e5a49e825e 100644 --- a/src/backend/commands/prepare.c +++ b/src/backend/commands/prepare.c @@ -582,6 +582,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, MemoryContextCounters mem_counters; MemoryContext planner_ctx = NULL; MemoryContext saved_ctx = NULL; + Size mem_consumed; if (es->memory) { @@ -596,6 +597,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, if (es->buffers) bufusage_start = pgBufferUsage; INSTR_TIME_SET_CURRENT(planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext); /* Look it up in the hash table */ entry = FetchPreparedStatement(execstmt->name, true); @@ -632,6 +634,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, INSTR_TIME_SET_CURRENT(planduration); INSTR_TIME_SUBTRACT(planduration, planstart); + mem_consumed = MemoryContextMemUsed(CurrentMemoryContext) - mem_consumed; if (es->memory) { @@ -656,7 +659,7 @@ ExplainExecuteQuery(ExecuteStmt *execstmt, IntoClause *into, ExplainState *es, if (pstmt->commandType != CMD_UTILITY) ExplainOnePlan(pstmt, into, es, query_string, paramLI, pstate->p_queryEnv, &planduration, (es->buffers ? &bufusage : NULL), - es->memory ? &mem_counters : NULL); + es->memory ? &mem_counters : NULL, &mem_consumed); else ExplainOneUtility(pstmt->utilityStmt, into, es, pstate, paramLI); diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index 70d33226cb..e0a119a886 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -865,6 +865,25 @@ MemoryContextStatsDetail(MemoryContext context, } } +/* + * Compute the memory used in the given context and its children. + * + * XXX: Instead of this separate function we could modify + * MemoryContextStatsDetail() to report used memory and disable printing the + * detailed stats. + */ +extern Size +MemoryContextMemUsed(MemoryContext context) +{ + MemoryContextCounters grand_totals; + + memset(&grand_totals, 0, sizeof(grand_totals)); + + MemoryContextStatsInternal(context, 0, false, 100, &grand_totals, false); + + return grand_totals.totalspace - grand_totals.freespace; +} + /* * MemoryContextStatsInternal * One recursion level for MemoryContextStats diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h index aa5872bc15..4fb267afef 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -108,7 +108,8 @@ extern void ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ParamListInfo params, QueryEnvironment *queryEnv, const instr_time *planduration, const BufferUsage *bufusage, - const MemoryContextCounters *mem_counters); + const MemoryContextCounters *mem_counters, + const Size *mem_consumed); extern void ExplainPrintPlan(ExplainState *es, QueryDesc *queryDesc); extern void ExplainPrintTriggers(ExplainState *es, QueryDesc *queryDesc); diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h index bf93433b78..9afb0bfb2a 100644 --- a/src/include/utils/memutils.h +++ b/src/include/utils/memutils.h @@ -92,6 +92,7 @@ extern void MemoryContextStatsDetail(MemoryContext context, bool print_to_stderr); extern void MemoryContextAllowInCriticalSection(MemoryContext context, bool allow); +extern Size MemoryContextMemUsed(MemoryContext context); #ifdef MEMORY_CONTEXT_CHECKING extern void MemoryContextCheck(MemoryContext context); -- 2.45.2.windows.1 [text/plain] remove-some-assertions.txt (2.2K, ../../CAJ2pMkY-Vd55REn3N1ACuRS0UT5QEe38r2pJ6=MatLKZkXHOxg@mail.gmail.com/3-remove-some-assertions.txt) download | inline diff: diff --git a/src/backend/optimizer/path/equivclass.c b/src/backend/optimizer/path/equivclass.c index 86057538c5..9bbe11a9de 100644 --- a/src/backend/optimizer/path/equivclass.c +++ b/src/backend/optimizer/path/equivclass.c @@ -4014,18 +4014,6 @@ get_ec_source_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) rel_esis = bms_add_members(rel_esis, index->source_indexes); } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(rel_esis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, - i); - - Assert(bms_overlap(relids, rinfo->clause_relids)); - } -#endif - /* bitwise-AND to leave only the ones for this EquivalenceClass */ return bms_int_members(rel_esis, ec->ec_source_indexes); } @@ -4065,18 +4053,6 @@ get_ec_source_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, } } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(esis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_sources, - i); - - Assert(bms_is_subset(relids, rinfo->clause_relids)); - } -#endif - return esis; } @@ -4103,18 +4079,6 @@ get_ec_derive_indexes(PlannerInfo *root, EquivalenceClass *ec, Relids relids) rel_edis = bms_add_members(rel_edis, index->derive_indexes); } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(rel_edis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, - i); - - Assert(bms_overlap(relids, rinfo->clause_relids)); - } -#endif - /* bitwise-AND to leave only the ones for this EquivalenceClass */ return bms_int_members(rel_edis, ec->ec_derive_indexes); } @@ -4154,18 +4118,6 @@ get_ec_derive_indexes_strict(PlannerInfo *root, EquivalenceClass *ec, } } -#ifdef USE_ASSERT_CHECKING - /* verify the results look sane */ - i = -1; - while ((i = bms_next_member(edis, i)) >= 0) - { - RestrictInfo *rinfo = list_nth_node(RestrictInfo, root->eq_derives, - i); - - Assert(bms_is_subset(relids, rinfo->clause_relids)); - } -#endif - return edis; } ^ permalink raw reply [nested|flat] 21+ messages in thread
end of thread, other threads:[~2024-12-11 03:16 UTC | newest] Thread overview: 21+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-11-04 01:08 [PATCH 05/15] Maybe better than looping twice. For partitioned tables, create only "inherited" stats_ext. Justin Pryzby <[email protected]> 2023-11-22 05:32 Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2023-11-30 04:18 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2023-12-13 06:21 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2023-12-16 15:41 ` Re: [PoC] Reducing planning time when tables have many partitions Alena Rybakina <[email protected]> 2024-01-17 09:33 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-02-12 21:19 ` Re: [PoC] Reducing planning time when tables have many partitions Alena Rybakina <[email protected]> 2024-02-28 11:18 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-03-06 14:16 ` Re: [PoC] Reducing planning time when tables have many partitions Ashutosh Bapat <[email protected]> 2024-05-02 07:56 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-05-02 14:35 ` Re: [PoC] Reducing planning time when tables have many partitions jian he <[email protected]> 2024-05-16 02:44 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-08-29 05:34 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-10-01 02:35 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-10-15 03:20 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-11-27 19:51 ` Re: [PoC] Reducing planning time when tables have many partitions Dmitry Dolgov <[email protected]> 2024-12-02 08:51 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]> 2024-12-03 04:12 ` Re: [PoC] Reducing planning time when tables have many partitions Ashutosh Bapat <[email protected]> 2024-12-03 10:38 ` Re: [PoC] Reducing planning time when tables have many partitions Alvaro Herrera <[email protected]> 2024-12-03 12:10 ` Re: [PoC] Reducing planning time when tables have many partitions Ashutosh Bapat <[email protected]> 2024-12-11 03:16 ` Re: [PoC] Reducing planning time when tables have many partitions Yuya Watari <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox