agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v33 1/2] Unique key 11+ messages / 3 participants [nested] [flat]
* [PATCH v33 1/2] Unique key @ 2019-07-09 10:44 jesperpedersen <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: jesperpedersen @ 2019-07-09 10:44 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/indxpath.c | 41 +++++++ src/backend/optimizer/path/pathkeys.c | 71 ++++++++++-- src/backend/optimizer/path/uniquekey.c | 147 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 17 ++- src/backend/optimizer/util/pathnode.c | 12 ++ src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 18 +++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 1 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 373 insertions(+), 13 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d76fae44b8..16083e7a7e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1723,6 +1723,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2205,6 +2206,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2214,6 +2216,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2401,6 +2404,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4092,6 +4103,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d8..d286b34544 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS = \ joinpath.o \ joinrels.o \ pathkeys.o \ - tidpath.o + tidpath.o \ + uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8286d9cf34..bbc13e6141 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 2a50272da6..bd1ea53e5c 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys = NIL; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 71b9d42c99..054df9a617 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Checks if the new pathkey's equivalence class is the same as that of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c new file mode 100644 index 0000000000..13d4ebb98c --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------- + * + * uniquekey.c + * Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceClass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ + List *result = NIL; + List *sortkeys; + ListCell *l; + + sortkeys = make_pathkeys_for_uniquekeys(root, + sortclauses, + root->processed_tlist); + + /* Create a uniquekey and add it to the list */ + foreach(l, sortkeys) + { + PathKey *pathkey = (PathKey *) lfirst(l); + EquivalenceClass *ec = pathkey->pk_eclass; + UniqueKey *unique_key = make_canonical_uniquekey(root, ec); + + result = lappend(result, unique_key); + } + + return result; +} + +/* + * uniquekeys_contained_in + * Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ + ListCell *key1, + *key2; + + /* + * Fall out quickly if we are passed two identical lists. This mostly + * catches the case where both are NIL, but that's common enough to + * warrant the test. + */ + if (keys1 == keys2) + return true; + + foreach(key2, keys2) + { + bool found = false; + UniqueKey *uniquekey2 = (UniqueKey *) lfirst(key2); + + foreach(key1, keys1) + { + UniqueKey *uniquekey1 = (UniqueKey *) lfirst(key1); + + if (uniquekey1->eq_clause == uniquekey2->eq_clause) + { + found = true; + break; + } + } + + if (!found) + return false; + } + + return true; +} + +/* + * has_useful_uniquekeys + * Detect whether the planner could have any uniquekeys that are + * useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ + if (root->query_uniquekeys != NIL) + return true; /* there are some */ + return false; /* definitely useless */ +} + +/* + * make_canonical_uniquekey + * Given the parameters for a UniqueKey, find any pre-existing matching + * uniquekey in the query's list of "canonical" uniquekeys. Make a new + * entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instead, + * equivclass.c will complain if a merge occurs after root->canon_uniquekeys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, + EquivalenceClass *eclass) +{ + UniqueKey *uk; + ListCell *lc; + MemoryContext oldcontext; + + /* The passed eclass might be non-canonical, so chase up to the top */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + foreach(lc, root->canon_uniquekeys) + { + uk = (UniqueKey *) lfirst(lc); + if (eclass == uk->eq_clause) + return uk; + } + + /* + * Be sure canonical uniquekeys are allocated in the main planning context. + * Not an issue in normal planning, but it is for GEQO. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + + uk = makeNode(UniqueKey); + uk->eq_clause = eclass; + + root->canon_uniquekeys = lappend(root->canon_uniquekeys, uk); + + MemoryContextSwitchTo(oldcontext); + + return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 8634940efc..dd64775d8f 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; + root->query_uniquekeys = NIL; } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 62dfc6d44a..3a372af91b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; + root->canon_uniquekeys = NIL; root->left_join_clauses = NIL; root->right_join_clauses = NIL; root->full_join_clauses = NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d6f2153593..984fca0696 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3657,15 +3657,30 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6237,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index e6d08aede5..a006dbbe9c 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -940,6 +940,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -964,6 +965,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1000,6 +1002,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1018,6 +1021,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1061,6 +1065,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1922,6 +1927,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1948,6 +1954,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1974,6 +1981,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -1999,6 +2007,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2025,6 +2034,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2051,6 +2061,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2077,6 +2088,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index baced7eec0..a1511b46ea 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3d3be197e0..4e329f0fb5 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* unique keys used for the query */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -1077,6 +1081,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1119,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1145,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index 6126b491bf..006248bfb5 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e450fe112a..f75ff6f323 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -44,6 +44,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9ab73bd20c..5b6be383b3 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.0 --ezqb2fwnenb5aigz Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v33-0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH] Unique key @ 2019-07-09 10:44 jesperpedersen <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: jesperpedersen @ 2019-07-09 10:44 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/indxpath.c | 41 +++++++ src/backend/optimizer/path/pathkeys.c | 71 ++++++++++-- src/backend/optimizer/path/uniquekey.c | 147 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 17 ++- src/backend/optimizer/util/pathnode.c | 12 ++ src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 18 +++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 1 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 373 insertions(+), 13 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index b0dcd02ff6..1ccd68d3aa 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1720,6 +1720,7 @@ _outPathInfo(StringInfo str, const Path *node) =09WRITE_FLOAT_FIELD(startup_cost, "%.2f"); =09WRITE_FLOAT_FIELD(total_cost, "%.2f"); =09WRITE_NODE_FIELD(pathkeys); +=09WRITE_NODE_FIELD(uniquekeys); } =20 /* @@ -2201,6 +2202,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *no= de) =09WRITE_NODE_FIELD(eq_classes); =09WRITE_BOOL_FIELD(ec_merging_done); =09WRITE_NODE_FIELD(canon_pathkeys); +=09WRITE_NODE_FIELD(canon_uniquekeys); =09WRITE_NODE_FIELD(left_join_clauses); =09WRITE_NODE_FIELD(right_join_clauses); =09WRITE_NODE_FIELD(full_join_clauses); @@ -2210,6 +2212,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *no= de) =09WRITE_NODE_FIELD(placeholder_list); =09WRITE_NODE_FIELD(fkey_list); =09WRITE_NODE_FIELD(query_pathkeys); +=09WRITE_NODE_FIELD(query_uniquekeys); =09WRITE_NODE_FIELD(group_pathkeys); =09WRITE_NODE_FIELD(window_pathkeys); =09WRITE_NODE_FIELD(distinct_pathkeys); @@ -2397,6 +2400,14 @@ _outPathKey(StringInfo str, const PathKey *node) =09WRITE_BOOL_FIELD(pk_nulls_first); } =20 +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ +=09WRITE_NODE_TYPE("UNIQUEKEY"); + +=09WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4083,6 +4094,9 @@ outNode(StringInfo str, const void *obj) =09=09=09case T_PathKey: =09=09=09=09_outPathKey(str, obj); =09=09=09=09break; +=09=09=09case T_UniqueKey: +=09=09=09=09_outUniqueKey(str, obj); +=09=09=09=09break; =09=09=09case T_PathTarget: =09=09=09=09_outPathTarget(str, obj); =09=09=09=09break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 4ecde6b421..435b32063c 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtabl= e) =09printf(")\n"); } =20 +/* + * print_uniquekeys - + *=09 uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ +=09ListCell *l; + +=09printf("("); +=09foreach(l, uniquekeys) +=09{ +=09=09UniqueKey *unique_key =3D (UniqueKey *) lfirst(l); +=09=09EquivalenceClass *eclass =3D (EquivalenceClass *) unique_key->eq_cla= use; +=09=09ListCell *k; +=09=09bool=09=09first =3D true; + +=09=09/* chase up */ +=09=09while (eclass->ec_merged) +=09=09=09eclass =3D eclass->ec_merged; + +=09=09printf("("); +=09=09foreach(k, eclass->ec_members) +=09=09{ +=09=09=09EquivalenceMember *mem =3D (EquivalenceMember *) lfirst(k); + +=09=09=09if (first) +=09=09=09=09first =3D false; +=09=09=09else +=09=09=09=09printf(", "); +=09=09=09print_expr((Node *) mem->em_expr, rtable); +=09=09} +=09=09printf(")"); +=09=09if (lnext(uniquekeys, l)) +=09=09=09printf(", "); +=09} +=09printf(")\n"); +} + /* * print_tl *=09 print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/pa= th/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS =3D \ =09joinpath.o \ =09joinrels.o \ =09pathkeys.o \ -=09tidpath.o +=09tidpath.o \ +=09uniquekey.o =20 include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/= path/allpaths.c index db3a68a51d..5fc9b81746 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent= ) =09=09print_pathkeys(path->pathkeys, root->parse->rtable); =09} =20 +=09if (path->uniquekeys) +=09{ +=09=09for (i =3D 0; i < indent; i++) +=09=09=09printf("\t"); +=09=09printf(" uniquekeys: "); +=09=09print_uniquekeys(path->uniquekeys, root->parse->rtable); +=09} + =09if (join) =09{ =09=09JoinPath *jp =3D (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/= path/indxpath.c index 37b257cd0e..88c1dd0f59 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *= index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09 EquivalenceClass *ec, EquivalenceMember *em, =09=09=09=09=09=09=09=09=09 void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); =20 =20 /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, =09List=09 *orderbyclausecols; =09List=09 *index_pathkeys; =09List=09 *useful_pathkeys; +=09List=09 *useful_uniquekeys =3D NIL; =09bool=09=09found_lower_saop_clause; =09bool=09=09pathkeys_possibly_useful; =09bool=09=09index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *re= l, =09if (index_clauses !=3D NIL || useful_pathkeys !=3D NIL || useful_predic= ate || =09=09index_only_scan) =09{ +=09=09if (has_useful_uniquekeys(root)) +=09=09=09useful_uniquekeys =3D get_uniquekeys_for_index(root, useful_pathk= eys); + =09=09ipath =3D create_index_path(root, index, =09=09=09=09=09=09=09=09 index_clauses, =09=09=09=09=09=09=09=09 orderbyclauses, =09=09=09=09=09=09=09=09 orderbyclausecols, =09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09 index_is_ordered ? =09=09=09=09=09=09=09=09 ForwardScanDirection : =09=09=09=09=09=09=09=09 NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09 orderbyclauses, =09=09=09=09=09=09=09=09=09 orderbyclausecols, =09=09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09=09 index_is_ordered ? =09=09=09=09=09=09=09=09=09 ForwardScanDirection : =09=09=09=09=09=09=09=09=09 NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *re= l, =09=09=09=09=09=09=09=09=09=09=09=09=09index_pathkeys); =09=09if (useful_pathkeys !=3D NIL) =09=09{ +=09=09=09if (has_useful_uniquekeys(root)) +=09=09=09=09useful_uniquekeys =3D get_uniquekeys_for_index(root, useful_pa= thkeys); + =09=09=09ipath =3D create_index_path(root, index, =09=09=09=09=09=09=09=09=09 index_clauses, =09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09=09 BackwardScanDirection, =09=09=09=09=09=09=09=09=09 index_only_scan, =09=09=09=09=09=09=09=09=09 outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09=09=09 BackwardScanDirection, =09=09=09=09=09=09=09=09=09=09 index_only_scan, =09=09=09=09=09=09=09=09=09=09 outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, =09return clause; } =20 +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ +=09ListCell *lc; + +=09if (pathkeys) +=09{ +=09=09List *uniquekeys =3D NIL; +=09=09foreach(lc, pathkeys) +=09=09{ +=09=09=09UniqueKey *unique_key; +=09=09=09PathKey *pk =3D (PathKey *) lfirst(lc); +=09=09=09EquivalenceClass *ec =3D (EquivalenceClass *) pk->pk_eclass; + +=09=09=09unique_key =3D makeNode(UniqueKey); +=09=09=09unique_key->eq_clause =3D ec; + +=09=09=09lappend(uniquekeys, unique_key); +=09=09} + +=09=09if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) +=09=09=09return uniquekeys; +=09} + +=09return NIL; +} =20 /*************************************************************************= *** *=09=09=09=09---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS=09---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/= path/pathkeys.c index 2f4fea241a..b0dc1bc22a 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" =20 =20 +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, =09=09=09=09=09=09=09=09=09=09=09 RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, =09return pk; } =20 +/* + * pathkey_is_unique + *=09 Checks if the new pathkey's equivalence class is the same as that = of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ +=09EquivalenceClass *new_ec =3D new_pathkey->pk_eclass; +=09ListCell *lc; + +=09/* If same EC already is already in the list, then not unique */ +=09foreach(lc, pathkeys) +=09{ +=09=09PathKey *old_pathkey =3D (PathKey *) lfirst(lc); + +=09=09if (new_ec =3D=3D old_pathkey->pk_eclass) +=09=09=09return false; +=09} + +=09return true; +} + /* * pathkey_is_redundant *=09 Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { =09EquivalenceClass *new_ec =3D new_pathkey->pk_eclass; -=09ListCell *lc; =20 =09/* Check for EC containing a constant --- unconditionally redundant */ =09if (EC_MUST_BE_REDUNDANT(new_ec)) =09=09return true; =20 -=09/* If same EC already used in list, then redundant */ -=09foreach(lc, pathkeys) -=09{ -=09=09PathKey *old_pathkey =3D (PathKey *) lfirst(lc); - -=09=09if (new_ec =3D=3D old_pathkey->pk_eclass) -=09=09=09return true; -=09} - -=09return false; +=09return !pathkey_is_unique(new_pathkey, pathkeys); } =20 /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, =09return pathkeys; } =20 +/* + * make_pathkeys_for_uniquekeyclauses + *=09=09Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, +=09=09=09=09=09=09=09 List *sortclauses, +=09=09=09=09=09=09=09 List *tlist) +{ +=09List=09 *pathkeys =3D NIL; +=09ListCell *l; + +=09foreach(l, sortclauses) +=09{ +=09=09SortGroupClause *sortcl =3D (SortGroupClause *) lfirst(l); +=09=09Expr=09 *sortkey; +=09=09PathKey *pathkey; + +=09=09sortkey =3D (Expr *) get_sortgroupclause_expr(sortcl, tlist); +=09=09Assert(OidIsValid(sortcl->sortop)); +=09=09pathkey =3D make_pathkey_from_sortop(root, +=09=09=09=09=09=09=09=09=09=09 sortkey, +=09=09=09=09=09=09=09=09=09=09 root->nullable_baserels, +=09=09=09=09=09=09=09=09=09=09 sortcl->sortop, +=09=09=09=09=09=09=09=09=09=09 sortcl->nulls_first, +=09=09=09=09=09=09=09=09=09=09 sortcl->tleSortGroupRef, +=09=09=09=09=09=09=09=09=09=09 true); + +=09=09if (pathkey_is_unique(pathkey, pathkeys)) +=09=09=09pathkeys =3D lappend(pathkeys, pathkey); +=09} + +=09return pathkeys; +} + /*************************************************************************= *** *=09=09PATHKEYS AND MERGECLAUSES *************************************************************************= ***/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer= /path/uniquekey.c new file mode 100644 index 0000000000..13d4ebb98c --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------= - + * + * uniquekey.c + *=09 Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + *=09 src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------= - + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceC= lass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ +=09List *result =3D NIL; +=09List *sortkeys; +=09ListCell *l; + +=09sortkeys =3D make_pathkeys_for_uniquekeys(root, +=09=09=09=09=09=09=09=09=09=09=09sortclauses, +=09=09=09=09=09=09=09=09=09=09=09root->processed_tlist); + +=09/* Create a uniquekey and add it to the list */ +=09foreach(l, sortkeys) +=09{ +=09=09PathKey *pathkey =3D (PathKey *) lfirst(l); +=09=09EquivalenceClass *ec =3D pathkey->pk_eclass; +=09=09UniqueKey *unique_key =3D make_canonical_uniquekey(root, ec); + +=09=09result =3D lappend(result, unique_key); +=09} + +=09return result; +} + +/* + * uniquekeys_contained_in + *=09 Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ +=09ListCell *key1, +=09=09=09 *key2; + +=09/* +=09 * Fall out quickly if we are passed two identical lists. This mostly +=09 * catches the case where both are NIL, but that's common enough to +=09 * warrant the test. +=09 */ +=09if (keys1 =3D=3D keys2) +=09=09return true; + +=09foreach(key2, keys2) +=09{ +=09=09bool found =3D false; +=09=09UniqueKey *uniquekey2 =3D (UniqueKey *) lfirst(key2); + +=09=09foreach(key1, keys1) +=09=09{ +=09=09=09UniqueKey *uniquekey1 =3D (UniqueKey *) lfirst(key1); + +=09=09=09if (uniquekey1->eq_clause =3D=3D uniquekey2->eq_clause) +=09=09=09{ +=09=09=09=09found =3D true; +=09=09=09=09break; +=09=09=09} +=09=09} + +=09=09if (!found) +=09=09=09return false; +=09} + +=09return true; +} + +/* + * has_useful_uniquekeys + *=09=09Detect whether the planner could have any uniquekeys that are + *=09=09useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ +=09if (root->query_uniquekeys !=3D NIL) +=09=09return true;=09/* there are some */ +=09return false;=09=09/* definitely useless */ +} + +/* + * make_canonical_uniquekey + *=09 Given the parameters for a UniqueKey, find any pre-existing matchin= g + *=09 uniquekey in the query's list of "canonical" uniquekeys. Make a ne= w + *=09 entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instea= d, + * equivclass.c will complain if a merge occurs after root->canon_uniqueke= ys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, +=09=09=09=09=09=09 EquivalenceClass *eclass) +{ +=09UniqueKey *uk; +=09ListCell *lc; +=09MemoryContext oldcontext; + +=09/* The passed eclass might be non-canonical, so chase up to the top */ +=09while (eclass->ec_merged) +=09=09eclass =3D eclass->ec_merged; + +=09foreach(lc, root->canon_uniquekeys) +=09{ +=09=09uk =3D (UniqueKey *) lfirst(lc); +=09=09if (eclass =3D=3D uk->eq_clause) +=09=09=09return uk; +=09} + +=09/* +=09 * Be sure canonical uniquekeys are allocated in the main planning cont= ext. +=09 * Not an issue in normal planning, but it is for GEQO. +=09 */ +=09oldcontext =3D MemoryContextSwitchTo(root->planner_cxt); + +=09uk =3D makeNode(UniqueKey); +=09uk->eq_clause =3D eclass; + +=09root->canon_uniquekeys =3D lappend(root->canon_uniquekeys, uk); + +=09MemoryContextSwitchTo(oldcontext); + +=09return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/p= lan/planagg.c index 974f6204ca..f6144b2267 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) =09=09=09=09=09=09=09=09=09 root->parse->targetList); =20 =09root->query_pathkeys =3D root->sort_pathkeys; +=09root->query_uniquekeys =3D NIL; } =20 /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/= plan/planmain.c index f0c1b52a2e..3ccde03ab7 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, =09root->join_rel_level =3D NULL; =09root->join_cur_level =3D 0; =09root->canon_pathkeys =3D NIL; +=09root->canon_uniquekeys =3D NIL; =09root->left_join_clauses =3D NIL; =09root->right_join_clauses =3D NIL; =09root->full_join_clauses =3D NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/p= lan/planner.c index 7fe11b59a0..8f03a20825 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3651,15 +3651,30 @@ standard_qp_callback(PlannerInfo *root, void *extra= ) =09 * much easier, since we know that the parser ensured that one is a =09 * superset of the other. =09 */ +=09root->query_uniquekeys =3D NIL; + =09if (root->group_pathkeys) +=09{ =09=09root->query_pathkeys =3D root->group_pathkeys; + +=09=09if (!root->parse->hasAggs) +=09=09=09root->query_uniquekeys =3D build_uniquekeys(root, qp_extra->group= Clause); +=09} =09else if (root->window_pathkeys) =09=09root->query_pathkeys =3D root->window_pathkeys; =09else if (list_length(root->distinct_pathkeys) > =09=09=09 list_length(root->sort_pathkeys)) +=09{ =09=09root->query_pathkeys =3D root->distinct_pathkeys; +=09=09root->query_uniquekeys =3D build_uniquekeys(root, parse->distinctCla= use); +=09} =09else if (root->sort_pathkeys) +=09{ =09=09root->query_pathkeys =3D root->sort_pathkeys; + +=09=09if (root->distinct_pathkeys) +=09=09=09root->query_uniquekeys =3D build_uniquekeys(root, parse->distinct= Clause); +=09} =09else =09=09root->query_pathkeys =3D NIL; } @@ -6216,7 +6231,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) =20 =09/* Estimate the cost of index scan */ =09indexScanPath =3D create_index_path(root, indexInfo, -=09=09=09=09=09=09=09=09=09 NIL, NIL, NIL, NIL, +=09=09=09=09=09=09=09=09=09 NIL, NIL, NIL, NIL, NIL, =09=09=09=09=09=09=09=09=09 ForwardScanDirection, false, =09=09=09=09=09=09=09=09=09 NULL, 1.0, false); =20 diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/= util/pathnode.c index 60c93ee7c5..ec02c468d0 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -940,6 +940,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D parallel_workers; =09pathnode->pathkeys =3D NIL;=09/* seqscan has unordered result */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_seqscan(pathnode, root, rel, pathnode->param_info); =20 @@ -964,6 +965,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *r= el, Relids required_outer =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* samplescan has unordered result */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_samplescan(pathnode, root, rel, pathnode->param_info); =20 @@ -1000,6 +1002,7 @@ create_index_path(PlannerInfo *root, =09=09=09=09 List *indexorderbys, =09=09=09=09 List *indexorderbycols, =09=09=09=09 List *pathkeys, +=09=09=09=09 List *uniquekeys, =09=09=09=09 ScanDirection indexscandir, =09=09=09=09 bool indexonly, =09=09=09=09 Relids required_outer, @@ -1018,6 +1021,7 @@ create_index_path(PlannerInfo *root, =09pathnode->path.parallel_safe =3D rel->consider_parallel; =09pathnode->path.parallel_workers =3D 0; =09pathnode->path.pathkeys =3D pathkeys; +=09pathnode->path.uniquekeys =3D uniquekeys; =20 =09pathnode->indexinfo =3D index; =09pathnode->indexclauses =3D indexclauses; @@ -1061,6 +1065,7 @@ create_bitmap_heap_path(PlannerInfo *root, =09pathnode->path.parallel_safe =3D rel->consider_parallel; =09pathnode->path.parallel_workers =3D parallel_degree; =09pathnode->path.pathkeys =3D NIL;=09/* always unordered */ +=09pathnode->path.uniquekeys =3D NIL; =20 =09pathnode->bitmapqual =3D bitmapqual; =20 @@ -1922,6 +1927,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInf= o *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D pathkeys; +=09pathnode->uniquekeys =3D NIL; =20 =09cost_functionscan(pathnode, root, rel, pathnode->param_info); =20 @@ -1948,6 +1954,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptIn= fo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); =20 @@ -1974,6 +1981,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo = *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_valuesscan(pathnode, root, rel, pathnode->param_info); =20 @@ -1999,6 +2007,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *re= l, Relids required_outer) =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* XXX for now, result is always unordere= d */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_ctescan(pathnode, root, rel, pathnode->param_info); =20 @@ -2025,6 +2034,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, Re= lOptInfo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); =20 @@ -2051,6 +2061,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo = *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_resultscan(pathnode, root, rel, pathnode->param_info); =20 @@ -2077,6 +2088,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptIn= fo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09/* Cost is the same as for a regular CTE scan */ =09cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index bce2d59b0d..cbb6ba2586 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag =09T_EquivalenceMember, =09T_PathKey, =09T_PathTarget, +=09T_UniqueKey, =09T_RestrictInfo, =09T_IndexClause, =09T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 23a06d718e..10ece6c875 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -267,6 +267,8 @@ struct PlannerInfo =20 =09List=09 *canon_pathkeys; /* list of "canonical" PathKeys */ =20 +=09List=09 *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + =09List=09 *left_join_clauses;=09/* list of RestrictInfos for mergejoina= ble =09=09=09=09=09=09=09=09=09 * outer join clauses w/nonnullable var on =09=09=09=09=09=09=09=09=09 * left */ @@ -295,6 +297,8 @@ struct PlannerInfo =20 =09List=09 *query_pathkeys; /* desired pathkeys for query_planner() */ =20 +=09List=09 *query_uniquekeys; /* unique keys used for the query */ + =09List=09 *group_pathkeys; /* groupClause pathkeys, if any */ =09List=09 *window_pathkeys;=09/* pathkeys of bottom window, if any */ =09List=09 *distinct_pathkeys;=09/* distinctClause pathkeys, if any */ @@ -1075,6 +1079,15 @@ typedef struct ParamPathInfo =09List=09 *ppi_clauses;=09/* join clauses available from outer rels */ } ParamPathInfo; =20 +/* + * UniqueKey + */ +typedef struct UniqueKey +{ +=09NodeTag=09=09type; + +=09EquivalenceClass *eq_clause;=09/* equivalence class */ +} UniqueKey; =20 /* * Type "Path" is used as-is for sequential-scan paths, as well as some ot= her @@ -1104,6 +1117,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1127,6 +1143,8 @@ typedef struct Path =20 =09List=09 *pathkeys;=09=09/* sort ordering of path's output */ =09/* pathkeys is a List of PathKey nodes; see above */ + +=09List=09 *uniquekeys;=09/* the unique keys, or NIL if none */ } Path; =20 /* Macro for extracting a path's parameterization relids; beware double ev= al */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index cbff56a724..f1a7112877 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); =20 diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathn= ode.h index a12af54971..37a946f857 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -44,6 +44,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, =09=09=09=09=09=09=09=09=09List *indexorderbys, =09=09=09=09=09=09=09=09=09List *indexorderbycols, =09=09=09=09=09=09=09=09=09List *pathkeys, +=09=09=09=09=09=09=09=09=09List *uniquekeys, =09=09=09=09=09=09=09=09=09ScanDirection indexscandir, =09=09=09=09=09=09=09=09=09bool indexonly, =09=09=09=09=09=09=09=09=09Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index c6c34630c2..c79e47eeaf 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, =09=09=09=09=09=09=09=09=09=09 List *sortclauses, =09=09=09=09=09=09=09=09=09=09 List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, +=09=09=09=09=09=09=09=09=09=09 List *sortclauses, +=09=09=09=09=09=09=09=09=09=09 List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, =09=09=09=09=09=09=09=09=09=09=09RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *ro= ot, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09List *live_childrels); =20 +/* + * uniquekey.c + *=09 Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif=09=09=09=09=09=09=09/* PATHS_H */ --=20 2.21.0 --------------51513615E9A6AD19C4BF8178 Content-Type: text/x-patch; charset=UTF-8; name="v30_0002-Index-skip-scan.patch" Content-Transfer-Encoding: quoted-printable Content-Disposition: attachment; filename="v30_0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH v31 1/2] Unique key @ 2019-07-09 10:44 jesperpedersen <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: jesperpedersen @ 2019-07-09 10:44 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/indxpath.c | 41 +++++++ src/backend/optimizer/path/pathkeys.c | 71 ++++++++++-- src/backend/optimizer/path/uniquekey.c | 147 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 17 ++- src/backend/optimizer/util/pathnode.c | 12 ++ src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 18 +++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 1 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 373 insertions(+), 13 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d76fae44b8..16083e7a7e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1723,6 +1723,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2205,6 +2206,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2214,6 +2216,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2401,6 +2404,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4092,6 +4103,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d8..d286b34544 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS = \ joinpath.o \ joinrels.o \ pathkeys.o \ - tidpath.o + tidpath.o \ + uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8286d9cf34..bbc13e6141 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 2a50272da6..bd1ea53e5c 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys = NIL; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 71b9d42c99..054df9a617 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Checks if the new pathkey's equivalence class is the same as that of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c new file mode 100644 index 0000000000..13d4ebb98c --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------- + * + * uniquekey.c + * Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceClass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ + List *result = NIL; + List *sortkeys; + ListCell *l; + + sortkeys = make_pathkeys_for_uniquekeys(root, + sortclauses, + root->processed_tlist); + + /* Create a uniquekey and add it to the list */ + foreach(l, sortkeys) + { + PathKey *pathkey = (PathKey *) lfirst(l); + EquivalenceClass *ec = pathkey->pk_eclass; + UniqueKey *unique_key = make_canonical_uniquekey(root, ec); + + result = lappend(result, unique_key); + } + + return result; +} + +/* + * uniquekeys_contained_in + * Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ + ListCell *key1, + *key2; + + /* + * Fall out quickly if we are passed two identical lists. This mostly + * catches the case where both are NIL, but that's common enough to + * warrant the test. + */ + if (keys1 == keys2) + return true; + + foreach(key2, keys2) + { + bool found = false; + UniqueKey *uniquekey2 = (UniqueKey *) lfirst(key2); + + foreach(key1, keys1) + { + UniqueKey *uniquekey1 = (UniqueKey *) lfirst(key1); + + if (uniquekey1->eq_clause == uniquekey2->eq_clause) + { + found = true; + break; + } + } + + if (!found) + return false; + } + + return true; +} + +/* + * has_useful_uniquekeys + * Detect whether the planner could have any uniquekeys that are + * useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ + if (root->query_uniquekeys != NIL) + return true; /* there are some */ + return false; /* definitely useless */ +} + +/* + * make_canonical_uniquekey + * Given the parameters for a UniqueKey, find any pre-existing matching + * uniquekey in the query's list of "canonical" uniquekeys. Make a new + * entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instead, + * equivclass.c will complain if a merge occurs after root->canon_uniquekeys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, + EquivalenceClass *eclass) +{ + UniqueKey *uk; + ListCell *lc; + MemoryContext oldcontext; + + /* The passed eclass might be non-canonical, so chase up to the top */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + foreach(lc, root->canon_uniquekeys) + { + uk = (UniqueKey *) lfirst(lc); + if (eclass == uk->eq_clause) + return uk; + } + + /* + * Be sure canonical uniquekeys are allocated in the main planning context. + * Not an issue in normal planning, but it is for GEQO. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + + uk = makeNode(UniqueKey); + uk->eq_clause = eclass; + + root->canon_uniquekeys = lappend(root->canon_uniquekeys, uk); + + MemoryContextSwitchTo(oldcontext); + + return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 8634940efc..dd64775d8f 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; + root->query_uniquekeys = NIL; } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 62dfc6d44a..3a372af91b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; + root->canon_uniquekeys = NIL; root->left_join_clauses = NIL; root->right_join_clauses = NIL; root->full_join_clauses = NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d6f2153593..984fca0696 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3657,15 +3657,30 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6237,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index e6d08aede5..a006dbbe9c 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -940,6 +940,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -964,6 +965,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1000,6 +1002,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1018,6 +1021,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1061,6 +1065,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1922,6 +1927,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1948,6 +1954,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1974,6 +1981,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -1999,6 +2007,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2025,6 +2034,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2051,6 +2061,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2077,6 +2088,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index baced7eec0..a1511b46ea 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3d3be197e0..4e329f0fb5 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* unique keys used for the query */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -1077,6 +1081,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1119,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1145,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index 6126b491bf..006248bfb5 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e450fe112a..f75ff6f323 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -44,6 +44,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9ab73bd20c..5b6be383b3 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.0 --cxnum6r77tfmznfe Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v31-0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH v32 1/2] Unique key @ 2019-07-09 10:44 jesperpedersen <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: jesperpedersen @ 2019-07-09 10:44 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/indxpath.c | 41 +++++++ src/backend/optimizer/path/pathkeys.c | 71 ++++++++++-- src/backend/optimizer/path/uniquekey.c | 147 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 17 ++- src/backend/optimizer/util/pathnode.c | 12 ++ src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 18 +++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 1 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 373 insertions(+), 13 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d76fae44b8..16083e7a7e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1723,6 +1723,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2205,6 +2206,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2214,6 +2216,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2401,6 +2404,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4092,6 +4103,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d8..d286b34544 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS = \ joinpath.o \ joinrels.o \ pathkeys.o \ - tidpath.o + tidpath.o \ + uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8286d9cf34..bbc13e6141 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 2a50272da6..bd1ea53e5c 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys = NIL; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 71b9d42c99..054df9a617 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Checks if the new pathkey's equivalence class is the same as that of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c new file mode 100644 index 0000000000..13d4ebb98c --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------- + * + * uniquekey.c + * Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceClass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ + List *result = NIL; + List *sortkeys; + ListCell *l; + + sortkeys = make_pathkeys_for_uniquekeys(root, + sortclauses, + root->processed_tlist); + + /* Create a uniquekey and add it to the list */ + foreach(l, sortkeys) + { + PathKey *pathkey = (PathKey *) lfirst(l); + EquivalenceClass *ec = pathkey->pk_eclass; + UniqueKey *unique_key = make_canonical_uniquekey(root, ec); + + result = lappend(result, unique_key); + } + + return result; +} + +/* + * uniquekeys_contained_in + * Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ + ListCell *key1, + *key2; + + /* + * Fall out quickly if we are passed two identical lists. This mostly + * catches the case where both are NIL, but that's common enough to + * warrant the test. + */ + if (keys1 == keys2) + return true; + + foreach(key2, keys2) + { + bool found = false; + UniqueKey *uniquekey2 = (UniqueKey *) lfirst(key2); + + foreach(key1, keys1) + { + UniqueKey *uniquekey1 = (UniqueKey *) lfirst(key1); + + if (uniquekey1->eq_clause == uniquekey2->eq_clause) + { + found = true; + break; + } + } + + if (!found) + return false; + } + + return true; +} + +/* + * has_useful_uniquekeys + * Detect whether the planner could have any uniquekeys that are + * useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ + if (root->query_uniquekeys != NIL) + return true; /* there are some */ + return false; /* definitely useless */ +} + +/* + * make_canonical_uniquekey + * Given the parameters for a UniqueKey, find any pre-existing matching + * uniquekey in the query's list of "canonical" uniquekeys. Make a new + * entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instead, + * equivclass.c will complain if a merge occurs after root->canon_uniquekeys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, + EquivalenceClass *eclass) +{ + UniqueKey *uk; + ListCell *lc; + MemoryContext oldcontext; + + /* The passed eclass might be non-canonical, so chase up to the top */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + foreach(lc, root->canon_uniquekeys) + { + uk = (UniqueKey *) lfirst(lc); + if (eclass == uk->eq_clause) + return uk; + } + + /* + * Be sure canonical uniquekeys are allocated in the main planning context. + * Not an issue in normal planning, but it is for GEQO. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + + uk = makeNode(UniqueKey); + uk->eq_clause = eclass; + + root->canon_uniquekeys = lappend(root->canon_uniquekeys, uk); + + MemoryContextSwitchTo(oldcontext); + + return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 8634940efc..dd64775d8f 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; + root->query_uniquekeys = NIL; } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 62dfc6d44a..3a372af91b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; + root->canon_uniquekeys = NIL; root->left_join_clauses = NIL; root->right_join_clauses = NIL; root->full_join_clauses = NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d6f2153593..984fca0696 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3657,15 +3657,30 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6237,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index e6d08aede5..a006dbbe9c 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -940,6 +940,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -964,6 +965,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1000,6 +1002,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1018,6 +1021,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1061,6 +1065,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1922,6 +1927,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1948,6 +1954,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1974,6 +1981,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -1999,6 +2007,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2025,6 +2034,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2051,6 +2061,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2077,6 +2088,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index baced7eec0..a1511b46ea 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3d3be197e0..4e329f0fb5 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* unique keys used for the query */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -1077,6 +1081,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1119,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1145,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index 6126b491bf..006248bfb5 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e450fe112a..f75ff6f323 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -44,6 +44,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9ab73bd20c..5b6be383b3 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.0 --f7z2si56qzkbcsr7 Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v32-0002-Index-skip-scan-visibility-check.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH 1/2] Unique key @ 2019-08-02 11:52 jesperpedersen <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: jesperpedersen @ 2019-08-02 11:52 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 2 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/costsize.c | 5 + src/backend/optimizer/path/indxpath.c | 41 +++++++ src/backend/optimizer/path/pathkeys.c | 72 ++++++++++-- src/backend/optimizer/path/uniquekey.c | 147 +++++++++++++++++++++++++ src/backend/optimizer/plan/planner.c | 18 ++- src/backend/optimizer/util/pathnode.c | 12 ++ src/backend/optimizer/util/tlist.c | 1 - src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 18 +++ src/include/nodes/print.h | 2 +- src/include/optimizer/pathnode.h | 1 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 377 insertions(+), 15 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index e6ce8e2110..9a4f3e8e4b 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1720,6 +1720,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2201,6 +2202,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2210,6 +2212,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2397,6 +2400,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4073,6 +4084,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 4ecde6b421..62c9d4ef7a 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * unique_key an UniqueKey + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 6864a62132..8249a6b6db 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -13,6 +13,6 @@ top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global OBJS = allpaths.o clausesel.o costsize.o equivclass.o indxpath.o \ - joinpath.o joinrels.o pathkeys.o tidpath.o + joinpath.o joinrels.o pathkeys.o tidpath.o uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index e9ee32b7f4..6f4c25f7dd 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3957,6 +3957,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 3a9a994733..2565dcf296 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -705,6 +705,11 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, path->path.parallel_aware = true; } + /* Consider cost based on unique key */ + if (path->path.uniquekeys) + { + } + /* * Now interpolate based on estimated index order correlation to get total * disk I/O cost for main table accesses. diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 5f339fdfde..4b90dd378a 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3369,6 +3381,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 2f4fea241a..0cba366c06 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,30 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Part of pathkey_is_redundant, that is reponsible for the case, when the + * new pathkey's equivalence class is the same as that of any existing + * member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +160,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1113,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c new file mode 100644 index 0000000000..13d4ebb98c --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------- + * + * uniquekey.c + * Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceClass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ + List *result = NIL; + List *sortkeys; + ListCell *l; + + sortkeys = make_pathkeys_for_uniquekeys(root, + sortclauses, + root->processed_tlist); + + /* Create a uniquekey and add it to the list */ + foreach(l, sortkeys) + { + PathKey *pathkey = (PathKey *) lfirst(l); + EquivalenceClass *ec = pathkey->pk_eclass; + UniqueKey *unique_key = make_canonical_uniquekey(root, ec); + + result = lappend(result, unique_key); + } + + return result; +} + +/* + * uniquekeys_contained_in + * Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ + ListCell *key1, + *key2; + + /* + * Fall out quickly if we are passed two identical lists. This mostly + * catches the case where both are NIL, but that's common enough to + * warrant the test. + */ + if (keys1 == keys2) + return true; + + foreach(key2, keys2) + { + bool found = false; + UniqueKey *uniquekey2 = (UniqueKey *) lfirst(key2); + + foreach(key1, keys1) + { + UniqueKey *uniquekey1 = (UniqueKey *) lfirst(key1); + + if (uniquekey1->eq_clause == uniquekey2->eq_clause) + { + found = true; + break; + } + } + + if (!found) + return false; + } + + return true; +} + +/* + * has_useful_uniquekeys + * Detect whether the planner could have any uniquekeys that are + * useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ + if (root->query_uniquekeys != NIL) + return true; /* there are some */ + return false; /* definitely useless */ +} + +/* + * make_canonical_uniquekey + * Given the parameters for a UniqueKey, find any pre-existing matching + * uniquekey in the query's list of "canonical" uniquekeys. Make a new + * entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instead, + * equivclass.c will complain if a merge occurs after root->canon_uniquekeys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, + EquivalenceClass *eclass) +{ + UniqueKey *uk; + ListCell *lc; + MemoryContext oldcontext; + + /* The passed eclass might be non-canonical, so chase up to the top */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + foreach(lc, root->canon_uniquekeys) + { + uk = (UniqueKey *) lfirst(lc); + if (eclass == uk->eq_clause) + return uk; + } + + /* + * Be sure canonical uniquekeys are allocated in the main planning context. + * Not an issue in normal planning, but it is for GEQO. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + + uk = makeNode(UniqueKey); + uk->eq_clause = eclass; + + root->canon_uniquekeys = lappend(root->canon_uniquekeys, uk); + + MemoryContextSwitchTo(oldcontext); + + return uk; +} diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 8f51f59f8a..5ee9ee6595 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3656,15 +3656,31 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->canon_uniquekeys = NIL; + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6238,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 0ac73984d2..ac0b937895 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -941,6 +941,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -965,6 +966,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1001,6 +1003,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1019,6 +1022,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1062,6 +1066,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1923,6 +1928,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1949,6 +1955,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1975,6 +1982,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -2000,6 +2008,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2026,6 +2035,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2052,6 +2062,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2078,6 +2089,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/backend/optimizer/util/tlist.c b/src/backend/optimizer/util/tlist.c index 7ccb10e4e1..618032e82c 100644 --- a/src/backend/optimizer/util/tlist.c +++ b/src/backend/optimizer/util/tlist.c @@ -427,7 +427,6 @@ get_sortgrouplist_exprs(List *sgClauses, List *targetList) return result; } - /***************************************************************************** * Functions to extract data from a list of SortGroupClauses * diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 4e2fb39105..a9b67c64f8 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index e3c579ee44..c1d6f33fc0 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -1077,6 +1081,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1119,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1145,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index cbff56a724..31e8f0686e 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -16,7 +16,6 @@ #include "executor/tuptable.h" - #define nodeDisplay(x) pprint(x) extern void print(const void *obj); @@ -28,6 +27,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 182ffeef4b..374cac157b 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -44,6 +44,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 7345137d1d..c7976d4a90 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -209,6 +209,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -235,4 +238,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.0 --------------A6AA3F98E061114DE0DF40DC Content-Type: text/x-patch; name="v23_0002-Index-skip-scan.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v23_0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH 1/2] Unique key @ 2019-11-11 13:49 jesperpedersen <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: jesperpedersen @ 2019-11-11 13:49 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/costsize.c | 5 + src/backend/optimizer/path/indxpath.c | 41 +++++++ src/backend/optimizer/path/pathkeys.c | 71 ++++++++++-- src/backend/optimizer/path/uniquekey.c | 147 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 17 ++- src/backend/optimizer/util/pathnode.c | 12 ++ src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 18 +++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 1 + src/include/optimizer/paths.h | 11 ++ 17 files changed, 378 insertions(+), 13 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index b0dcd02ff6..1ccd68d3aa 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1720,6 +1720,7 @@ _outPathInfo(StringInfo str, const Path *node) =09WRITE_FLOAT_FIELD(startup_cost, "%.2f"); =09WRITE_FLOAT_FIELD(total_cost, "%.2f"); =09WRITE_NODE_FIELD(pathkeys); +=09WRITE_NODE_FIELD(uniquekeys); } =20 /* @@ -2201,6 +2202,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *no= de) =09WRITE_NODE_FIELD(eq_classes); =09WRITE_BOOL_FIELD(ec_merging_done); =09WRITE_NODE_FIELD(canon_pathkeys); +=09WRITE_NODE_FIELD(canon_uniquekeys); =09WRITE_NODE_FIELD(left_join_clauses); =09WRITE_NODE_FIELD(right_join_clauses); =09WRITE_NODE_FIELD(full_join_clauses); @@ -2210,6 +2212,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *no= de) =09WRITE_NODE_FIELD(placeholder_list); =09WRITE_NODE_FIELD(fkey_list); =09WRITE_NODE_FIELD(query_pathkeys); +=09WRITE_NODE_FIELD(query_uniquekeys); =09WRITE_NODE_FIELD(group_pathkeys); =09WRITE_NODE_FIELD(window_pathkeys); =09WRITE_NODE_FIELD(distinct_pathkeys); @@ -2397,6 +2400,14 @@ _outPathKey(StringInfo str, const PathKey *node) =09WRITE_BOOL_FIELD(pk_nulls_first); } =20 +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ +=09WRITE_NODE_TYPE("UNIQUEKEY"); + +=09WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4083,6 +4094,9 @@ outNode(StringInfo str, const void *obj) =09=09=09case T_PathKey: =09=09=09=09_outPathKey(str, obj); =09=09=09=09break; +=09=09=09case T_UniqueKey: +=09=09=09=09_outUniqueKey(str, obj); +=09=09=09=09break; =09=09=09case T_PathTarget: =09=09=09=09_outPathTarget(str, obj); =09=09=09=09break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 4ecde6b421..62c9d4ef7a 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtabl= e) =09printf(")\n"); } =20 +/* + * print_uniquekeys - + *=09 unique_key an UniqueKey + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ +=09ListCell *l; + +=09printf("("); +=09foreach(l, uniquekeys) +=09{ +=09=09UniqueKey *unique_key =3D (UniqueKey *) lfirst(l); +=09=09EquivalenceClass *eclass =3D (EquivalenceClass *) unique_key->eq_cla= use; +=09=09ListCell *k; +=09=09bool=09=09first =3D true; + +=09=09/* chase up */ +=09=09while (eclass->ec_merged) +=09=09=09eclass =3D eclass->ec_merged; + +=09=09printf("("); +=09=09foreach(k, eclass->ec_members) +=09=09{ +=09=09=09EquivalenceMember *mem =3D (EquivalenceMember *) lfirst(k); + +=09=09=09if (first) +=09=09=09=09first =3D false; +=09=09=09else +=09=09=09=09printf(", "); +=09=09=09print_expr((Node *) mem->em_expr, rtable); +=09=09} +=09=09printf(")"); +=09=09if (lnext(uniquekeys, l)) +=09=09=09printf(", "); +=09} +=09printf(")\n"); +} + /* * print_tl *=09 print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/pa= th/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS =3D \ =09joinpath.o \ =09joinrels.o \ =09pathkeys.o \ -=09tidpath.o +=09tidpath.o \ +=09uniquekey.o =20 include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/= path/allpaths.c index db3a68a51d..5fc9b81746 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent= ) =09=09print_pathkeys(path->pathkeys, root->parse->rtable); =09} =20 +=09if (path->uniquekeys) +=09{ +=09=09for (i =3D 0; i < indent; i++) +=09=09=09printf("\t"); +=09=09printf(" uniquekeys: "); +=09=09print_uniquekeys(path->uniquekeys, root->parse->rtable); +=09} + =09if (join) =09{ =09=09JoinPath *jp =3D (JoinPath *) path; diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/= path/costsize.c index c5f6593485..0ec9a6db76 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -705,6 +705,11 @@ cost_index(IndexPath *path, PlannerInfo *root, double = loop_count, =09=09path->path.parallel_aware =3D true; =09} =20 +=09/* Consider cost based on unique key */ +=09if (path->path.uniquekeys) +=09{ +=09} + =09/* =09 * Now interpolate based on estimated index order correlation to get to= tal =09 * disk I/O cost for main table accesses. diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/= path/indxpath.c index 37b257cd0e..aa0da49119 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *= index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09 EquivalenceClass *ec, EquivalenceMember *em, =09=09=09=09=09=09=09=09=09 void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); =20 =20 /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, =09List=09 *orderbyclausecols; =09List=09 *index_pathkeys; =09List=09 *useful_pathkeys; +=09List=09 *useful_uniquekeys; =09bool=09=09found_lower_saop_clause; =09bool=09=09pathkeys_possibly_useful; =09bool=09=09index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *re= l, =09if (index_clauses !=3D NIL || useful_pathkeys !=3D NIL || useful_predic= ate || =09=09index_only_scan) =09{ +=09=09if (has_useful_uniquekeys(root)) +=09=09=09useful_uniquekeys =3D get_uniquekeys_for_index(root, useful_pathk= eys); + =09=09ipath =3D create_index_path(root, index, =09=09=09=09=09=09=09=09 index_clauses, =09=09=09=09=09=09=09=09 orderbyclauses, =09=09=09=09=09=09=09=09 orderbyclausecols, =09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09 index_is_ordered ? =09=09=09=09=09=09=09=09 ForwardScanDirection : =09=09=09=09=09=09=09=09 NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09 orderbyclauses, =09=09=09=09=09=09=09=09=09 orderbyclausecols, =09=09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09=09 index_is_ordered ? =09=09=09=09=09=09=09=09=09 ForwardScanDirection : =09=09=09=09=09=09=09=09=09 NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *re= l, =09=09=09=09=09=09=09=09=09=09=09=09=09index_pathkeys); =09=09if (useful_pathkeys !=3D NIL) =09=09{ +=09=09=09if (has_useful_uniquekeys(root)) +=09=09=09=09useful_uniquekeys =3D get_uniquekeys_for_index(root, useful_pa= thkeys); + =09=09=09ipath =3D create_index_path(root, index, =09=09=09=09=09=09=09=09=09 index_clauses, =09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09=09 BackwardScanDirection, =09=09=09=09=09=09=09=09=09 index_only_scan, =09=09=09=09=09=09=09=09=09 outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09=09 NIL, =09=09=09=09=09=09=09=09=09=09 useful_pathkeys, +=09=09=09=09=09=09=09=09=09=09 useful_uniquekeys, =09=09=09=09=09=09=09=09=09=09 BackwardScanDirection, =09=09=09=09=09=09=09=09=09=09 index_only_scan, =09=09=09=09=09=09=09=09=09=09 outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, =09return clause; } =20 +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ +=09ListCell *lc; + +=09if (pathkeys) +=09{ +=09=09List *uniquekeys =3D NIL; +=09=09foreach(lc, pathkeys) +=09=09{ +=09=09=09UniqueKey *unique_key; +=09=09=09PathKey *pk =3D (PathKey *) lfirst(lc); +=09=09=09EquivalenceClass *ec =3D (EquivalenceClass *) pk->pk_eclass; + +=09=09=09unique_key =3D makeNode(UniqueKey); +=09=09=09unique_key->eq_clause =3D ec; +=09=09=09 +=09=09=09lappend(uniquekeys, unique_key); +=09=09} + +=09=09if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) +=09=09=09return uniquekeys; +=09} + +=09return NIL; +} =20 /*************************************************************************= *** *=09=09=09=09---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS=09---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/= path/pathkeys.c index 2f4fea241a..b0dc1bc22a 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" =20 =20 +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, =09=09=09=09=09=09=09=09=09=09=09 RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, =09return pk; } =20 +/* + * pathkey_is_unique + *=09 Checks if the new pathkey's equivalence class is the same as that = of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ +=09EquivalenceClass *new_ec =3D new_pathkey->pk_eclass; +=09ListCell *lc; + +=09/* If same EC already is already in the list, then not unique */ +=09foreach(lc, pathkeys) +=09{ +=09=09PathKey *old_pathkey =3D (PathKey *) lfirst(lc); + +=09=09if (new_ec =3D=3D old_pathkey->pk_eclass) +=09=09=09return false; +=09} + +=09return true; +} + /* * pathkey_is_redundant *=09 Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { =09EquivalenceClass *new_ec =3D new_pathkey->pk_eclass; -=09ListCell *lc; =20 =09/* Check for EC containing a constant --- unconditionally redundant */ =09if (EC_MUST_BE_REDUNDANT(new_ec)) =09=09return true; =20 -=09/* If same EC already used in list, then redundant */ -=09foreach(lc, pathkeys) -=09{ -=09=09PathKey *old_pathkey =3D (PathKey *) lfirst(lc); - -=09=09if (new_ec =3D=3D old_pathkey->pk_eclass) -=09=09=09return true; -=09} - -=09return false; +=09return !pathkey_is_unique(new_pathkey, pathkeys); } =20 /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, =09return pathkeys; } =20 +/* + * make_pathkeys_for_uniquekeyclauses + *=09=09Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, +=09=09=09=09=09=09=09 List *sortclauses, +=09=09=09=09=09=09=09 List *tlist) +{ +=09List=09 *pathkeys =3D NIL; +=09ListCell *l; + +=09foreach(l, sortclauses) +=09{ +=09=09SortGroupClause *sortcl =3D (SortGroupClause *) lfirst(l); +=09=09Expr=09 *sortkey; +=09=09PathKey *pathkey; + +=09=09sortkey =3D (Expr *) get_sortgroupclause_expr(sortcl, tlist); +=09=09Assert(OidIsValid(sortcl->sortop)); +=09=09pathkey =3D make_pathkey_from_sortop(root, +=09=09=09=09=09=09=09=09=09=09 sortkey, +=09=09=09=09=09=09=09=09=09=09 root->nullable_baserels, +=09=09=09=09=09=09=09=09=09=09 sortcl->sortop, +=09=09=09=09=09=09=09=09=09=09 sortcl->nulls_first, +=09=09=09=09=09=09=09=09=09=09 sortcl->tleSortGroupRef, +=09=09=09=09=09=09=09=09=09=09 true); + +=09=09if (pathkey_is_unique(pathkey, pathkeys)) +=09=09=09pathkeys =3D lappend(pathkeys, pathkey); +=09} + +=09return pathkeys; +} + /*************************************************************************= *** *=09=09PATHKEYS AND MERGECLAUSES *************************************************************************= ***/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer= /path/uniquekey.c new file mode 100644 index 0000000000..13d4ebb98c --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------= - + * + * uniquekey.c + *=09 Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + *=09 src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------= - + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceC= lass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ +=09List *result =3D NIL; +=09List *sortkeys; +=09ListCell *l; + +=09sortkeys =3D make_pathkeys_for_uniquekeys(root, +=09=09=09=09=09=09=09=09=09=09=09sortclauses, +=09=09=09=09=09=09=09=09=09=09=09root->processed_tlist); + +=09/* Create a uniquekey and add it to the list */ +=09foreach(l, sortkeys) +=09{ +=09=09PathKey *pathkey =3D (PathKey *) lfirst(l); +=09=09EquivalenceClass *ec =3D pathkey->pk_eclass; +=09=09UniqueKey *unique_key =3D make_canonical_uniquekey(root, ec); + +=09=09result =3D lappend(result, unique_key); +=09} + +=09return result; +} + +/* + * uniquekeys_contained_in + *=09 Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ +=09ListCell *key1, +=09=09=09 *key2; + +=09/* +=09 * Fall out quickly if we are passed two identical lists. This mostly +=09 * catches the case where both are NIL, but that's common enough to +=09 * warrant the test. +=09 */ +=09if (keys1 =3D=3D keys2) +=09=09return true; + +=09foreach(key2, keys2) +=09{ +=09=09bool found =3D false; +=09=09UniqueKey *uniquekey2 =3D (UniqueKey *) lfirst(key2); + +=09=09foreach(key1, keys1) +=09=09{ +=09=09=09UniqueKey *uniquekey1 =3D (UniqueKey *) lfirst(key1); + +=09=09=09if (uniquekey1->eq_clause =3D=3D uniquekey2->eq_clause) +=09=09=09{ +=09=09=09=09found =3D true; +=09=09=09=09break; +=09=09=09} +=09=09} + +=09=09if (!found) +=09=09=09return false; +=09} + +=09return true; +} + +/* + * has_useful_uniquekeys + *=09=09Detect whether the planner could have any uniquekeys that are + *=09=09useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ +=09if (root->query_uniquekeys !=3D NIL) +=09=09return true;=09/* there are some */ +=09return false;=09=09/* definitely useless */ +} + +/* + * make_canonical_uniquekey + *=09 Given the parameters for a UniqueKey, find any pre-existing matchin= g + *=09 uniquekey in the query's list of "canonical" uniquekeys. Make a ne= w + *=09 entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instea= d, + * equivclass.c will complain if a merge occurs after root->canon_uniqueke= ys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, +=09=09=09=09=09=09 EquivalenceClass *eclass) +{ +=09UniqueKey *uk; +=09ListCell *lc; +=09MemoryContext oldcontext; + +=09/* The passed eclass might be non-canonical, so chase up to the top */ +=09while (eclass->ec_merged) +=09=09eclass =3D eclass->ec_merged; + +=09foreach(lc, root->canon_uniquekeys) +=09{ +=09=09uk =3D (UniqueKey *) lfirst(lc); +=09=09if (eclass =3D=3D uk->eq_clause) +=09=09=09return uk; +=09} + +=09/* +=09 * Be sure canonical uniquekeys are allocated in the main planning cont= ext. +=09 * Not an issue in normal planning, but it is for GEQO. +=09 */ +=09oldcontext =3D MemoryContextSwitchTo(root->planner_cxt); + +=09uk =3D makeNode(UniqueKey); +=09uk->eq_clause =3D eclass; + +=09root->canon_uniquekeys =3D lappend(root->canon_uniquekeys, uk); + +=09MemoryContextSwitchTo(oldcontext); + +=09return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/p= lan/planagg.c index 9381939c82..3d32f6dfd6 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -512,6 +512,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) =09=09=09=09=09=09=09=09=09 root->parse->targetList); =20 =09root->query_pathkeys =3D root->sort_pathkeys; +=09root->query_uniquekeys =3D NIL; } =20 /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/= plan/planmain.c index f0c1b52a2e..3ccde03ab7 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, =09root->join_rel_level =3D NULL; =09root->join_cur_level =3D 0; =09root->canon_pathkeys =3D NIL; +=09root->canon_uniquekeys =3D NIL; =09root->left_join_clauses =3D NIL; =09root->right_join_clauses =3D NIL; =09root->full_join_clauses =3D NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/p= lan/planner.c index 17c5f086fb..2507ec7d2a 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3652,15 +3652,30 @@ standard_qp_callback(PlannerInfo *root, void *extra= ) =09 * much easier, since we know that the parser ensured that one is a =09 * superset of the other. =09 */ +=09root->query_uniquekeys =3D NIL; + =09if (root->group_pathkeys) +=09{ =09=09root->query_pathkeys =3D root->group_pathkeys; + +=09=09if (!root->parse->hasAggs) +=09=09=09root->query_uniquekeys =3D build_uniquekeys(root, qp_extra->group= Clause); +=09} =09else if (root->window_pathkeys) =09=09root->query_pathkeys =3D root->window_pathkeys; =09else if (list_length(root->distinct_pathkeys) > =09=09=09 list_length(root->sort_pathkeys)) +=09{ =09=09root->query_pathkeys =3D root->distinct_pathkeys; +=09=09root->query_uniquekeys =3D build_uniquekeys(root, parse->distinctCla= use); +=09} =09else if (root->sort_pathkeys) +=09{ =09=09root->query_pathkeys =3D root->sort_pathkeys; + +=09=09if (root->distinct_pathkeys) +=09=09=09root->query_uniquekeys =3D build_uniquekeys(root, parse->distinct= Clause); +=09} =09else =09=09root->query_pathkeys =3D NIL; } @@ -6217,7 +6232,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) =20 =09/* Estimate the cost of index scan */ =09indexScanPath =3D create_index_path(root, indexInfo, -=09=09=09=09=09=09=09=09=09 NIL, NIL, NIL, NIL, +=09=09=09=09=09=09=09=09=09 NIL, NIL, NIL, NIL, NIL, =09=09=09=09=09=09=09=09=09 ForwardScanDirection, false, =09=09=09=09=09=09=09=09=09 NULL, 1.0, false); =20 diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/= util/pathnode.c index 34acb732ee..f268112386 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -941,6 +941,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D parallel_workers; =09pathnode->pathkeys =3D NIL;=09/* seqscan has unordered result */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_seqscan(pathnode, root, rel, pathnode->param_info); =20 @@ -965,6 +966,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *r= el, Relids required_outer =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* samplescan has unordered result */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_samplescan(pathnode, root, rel, pathnode->param_info); =20 @@ -1001,6 +1003,7 @@ create_index_path(PlannerInfo *root, =09=09=09=09 List *indexorderbys, =09=09=09=09 List *indexorderbycols, =09=09=09=09 List *pathkeys, +=09=09=09=09 List *uniquekeys, =09=09=09=09 ScanDirection indexscandir, =09=09=09=09 bool indexonly, =09=09=09=09 Relids required_outer, @@ -1019,6 +1022,7 @@ create_index_path(PlannerInfo *root, =09pathnode->path.parallel_safe =3D rel->consider_parallel; =09pathnode->path.parallel_workers =3D 0; =09pathnode->path.pathkeys =3D pathkeys; +=09pathnode->path.uniquekeys =3D uniquekeys; =20 =09pathnode->indexinfo =3D index; =09pathnode->indexclauses =3D indexclauses; @@ -1062,6 +1066,7 @@ create_bitmap_heap_path(PlannerInfo *root, =09pathnode->path.parallel_safe =3D rel->consider_parallel; =09pathnode->path.parallel_workers =3D parallel_degree; =09pathnode->path.pathkeys =3D NIL;=09/* always unordered */ +=09pathnode->path.uniquekeys =3D NIL; =20 =09pathnode->bitmapqual =3D bitmapqual; =20 @@ -1923,6 +1928,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInf= o *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D pathkeys; +=09pathnode->uniquekeys =3D NIL; =20 =09cost_functionscan(pathnode, root, rel, pathnode->param_info); =20 @@ -1949,6 +1955,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptIn= fo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); =20 @@ -1975,6 +1982,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo = *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_valuesscan(pathnode, root, rel, pathnode->param_info); =20 @@ -2000,6 +2008,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *re= l, Relids required_outer) =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* XXX for now, result is always unordere= d */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_ctescan(pathnode, root, rel, pathnode->param_info); =20 @@ -2026,6 +2035,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, Re= lOptInfo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); =20 @@ -2052,6 +2062,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo = *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09cost_resultscan(pathnode, root, rel, pathnode->param_info); =20 @@ -2078,6 +2089,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptIn= fo *rel, =09pathnode->parallel_safe =3D rel->consider_parallel; =09pathnode->parallel_workers =3D 0; =09pathnode->pathkeys =3D NIL;=09/* result is always unordered */ +=09pathnode->uniquekeys =3D NIL; =20 =09/* Cost is the same as for a regular CTE scan */ =09cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index bce2d59b0d..cbb6ba2586 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag =09T_EquivalenceMember, =09T_PathKey, =09T_PathTarget, +=09T_UniqueKey, =09T_RestrictInfo, =09T_IndexClause, =09T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 23a06d718e..6198c31cd4 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -267,6 +267,8 @@ struct PlannerInfo =20 =09List=09 *canon_pathkeys; /* list of "canonical" PathKeys */ =20 +=09List=09 *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + =09List=09 *left_join_clauses;=09/* list of RestrictInfos for mergejoina= ble =09=09=09=09=09=09=09=09=09 * outer join clauses w/nonnullable var on =09=09=09=09=09=09=09=09=09 * left */ @@ -295,6 +297,8 @@ struct PlannerInfo =20 =09List=09 *query_pathkeys; /* desired pathkeys for query_planner() */ =20 +=09List=09 *query_uniquekeys; /* */ + =09List=09 *group_pathkeys; /* groupClause pathkeys, if any */ =09List=09 *window_pathkeys;=09/* pathkeys of bottom window, if any */ =09List=09 *distinct_pathkeys;=09/* distinctClause pathkeys, if any */ @@ -1075,6 +1079,15 @@ typedef struct ParamPathInfo =09List=09 *ppi_clauses;=09/* join clauses available from outer rels */ } ParamPathInfo; =20 +/* + * UniqueKey + */ +typedef struct UniqueKey +{ +=09NodeTag=09=09type; + +=09EquivalenceClass *eq_clause;=09/* equivalence class */ +} UniqueKey; =20 /* * Type "Path" is used as-is for sequential-scan paths, as well as some ot= her @@ -1104,6 +1117,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1127,6 +1143,8 @@ typedef struct Path =20 =09List=09 *pathkeys;=09=09/* sort ordering of path's output */ =09/* pathkeys is a List of PathKey nodes; see above */ + +=09List=09 *uniquekeys;=09/* the unique keys, or NIL if none */ } Path; =20 /* Macro for extracting a path's parameterization relids; beware double ev= al */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index cbff56a724..f1a7112877 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); =20 diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathn= ode.h index a12af54971..37a946f857 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -44,6 +44,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, =09=09=09=09=09=09=09=09=09List *indexorderbys, =09=09=09=09=09=09=09=09=09List *indexorderbycols, =09=09=09=09=09=09=09=09=09List *pathkeys, +=09=09=09=09=09=09=09=09=09List *uniquekeys, =09=09=09=09=09=09=09=09=09ScanDirection indexscandir, =09=09=09=09=09=09=09=09=09bool indexonly, =09=09=09=09=09=09=09=09=09Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index c6c34630c2..c79e47eeaf 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, =09=09=09=09=09=09=09=09=09=09 List *sortclauses, =09=09=09=09=09=09=09=09=09=09 List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, +=09=09=09=09=09=09=09=09=09=09 List *sortclauses, +=09=09=09=09=09=09=09=09=09=09 List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, =09=09=09=09=09=09=09=09=09=09=09RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *ro= ot, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, =09=09=09=09=09=09=09=09=09List *live_childrels); =20 +/* + * uniquekey.c + *=09 Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif=09=09=09=09=09=09=09/* PATHS_H */ --=20 2.21.0 --------------25BA7482CCCC3CA4CEC0D330 Content-Type: text/x-patch; charset=UTF-8; name="v29_0002-Index-skip-scan.patch" Content-Transfer-Encoding: quoted-printable Content-Disposition: attachment; filename="v29_0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH 1/2] Unique key @ 2019-11-15 14:46 jesperpedersen <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: jesperpedersen @ 2019-11-15 14:46 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/indxpath.c | 41 +++++++ src/backend/optimizer/path/pathkeys.c | 71 ++++++++++-- src/backend/optimizer/path/uniquekey.c | 147 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 17 ++- src/backend/optimizer/util/pathnode.c | 12 ++ src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 18 +++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 1 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 373 insertions(+), 13 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d76fae44b8..16083e7a7e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1723,6 +1723,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2205,6 +2206,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2214,6 +2216,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2401,6 +2404,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4092,6 +4103,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d8..d286b34544 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS = \ joinpath.o \ joinrels.o \ pathkeys.o \ - tidpath.o + tidpath.o \ + uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8286d9cf34..bbc13e6141 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 2a50272da6..34e7fafa84 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys = NIL; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 71b9d42c99..054df9a617 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Checks if the new pathkey's equivalence class is the same as that of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c new file mode 100644 index 0000000000..13d4ebb98c --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,147 @@ +/*------------------------------------------------------------------------- + * + * uniquekey.c + * Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceClass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ + List *result = NIL; + List *sortkeys; + ListCell *l; + + sortkeys = make_pathkeys_for_uniquekeys(root, + sortclauses, + root->processed_tlist); + + /* Create a uniquekey and add it to the list */ + foreach(l, sortkeys) + { + PathKey *pathkey = (PathKey *) lfirst(l); + EquivalenceClass *ec = pathkey->pk_eclass; + UniqueKey *unique_key = make_canonical_uniquekey(root, ec); + + result = lappend(result, unique_key); + } + + return result; +} + +/* + * uniquekeys_contained_in + * Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ + ListCell *key1, + *key2; + + /* + * Fall out quickly if we are passed two identical lists. This mostly + * catches the case where both are NIL, but that's common enough to + * warrant the test. + */ + if (keys1 == keys2) + return true; + + foreach(key2, keys2) + { + bool found = false; + UniqueKey *uniquekey2 = (UniqueKey *) lfirst(key2); + + foreach(key1, keys1) + { + UniqueKey *uniquekey1 = (UniqueKey *) lfirst(key1); + + if (uniquekey1->eq_clause == uniquekey2->eq_clause) + { + found = true; + break; + } + } + + if (!found) + return false; + } + + return true; +} + +/* + * has_useful_uniquekeys + * Detect whether the planner could have any uniquekeys that are + * useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ + if (root->query_uniquekeys != NIL) + return true; /* there are some */ + return false; /* definitely useless */ +} + +/* + * make_canonical_uniquekey + * Given the parameters for a UniqueKey, find any pre-existing matching + * uniquekey in the query's list of "canonical" uniquekeys. Make a new + * entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instead, + * equivclass.c will complain if a merge occurs after root->canon_uniquekeys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, + EquivalenceClass *eclass) +{ + UniqueKey *uk; + ListCell *lc; + MemoryContext oldcontext; + + /* The passed eclass might be non-canonical, so chase up to the top */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + foreach(lc, root->canon_uniquekeys) + { + uk = (UniqueKey *) lfirst(lc); + if (eclass == uk->eq_clause) + return uk; + } + + /* + * Be sure canonical uniquekeys are allocated in the main planning context. + * Not an issue in normal planning, but it is for GEQO. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + + uk = makeNode(UniqueKey); + uk->eq_clause = eclass; + + root->canon_uniquekeys = lappend(root->canon_uniquekeys, uk); + + MemoryContextSwitchTo(oldcontext); + + return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 8634940efc..dd64775d8f 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; + root->query_uniquekeys = NIL; } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 62dfc6d44a..3a372af91b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; + root->canon_uniquekeys = NIL; root->left_join_clauses = NIL; root->right_join_clauses = NIL; root->full_join_clauses = NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d6f2153593..984fca0696 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3657,15 +3657,30 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6237,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index e6d08aede5..a006dbbe9c 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -940,6 +940,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -964,6 +965,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1000,6 +1002,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1018,6 +1021,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1061,6 +1065,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1922,6 +1927,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1948,6 +1954,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1974,6 +1981,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -1999,6 +2007,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2025,6 +2034,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2051,6 +2061,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2077,6 +2088,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index baced7eec0..a1511b46ea 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3d3be197e0..4e329f0fb5 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* unique keys used for the query */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -1077,6 +1081,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1119,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1145,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index 6126b491bf..006248bfb5 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e450fe112a..f75ff6f323 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -44,6 +44,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9ab73bd20c..5b6be383b3 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.1 --------------51B4E6A19513F7FD264CD0FF Content-Type: text/x-patch; charset=UTF-8; name="v31-0002-Index-skip-scan.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="v31-0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH v33 1/2] Unique key @ 2020-03-24 16:04 Dmitrii Dolgov <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: Dmitrii Dolgov @ 2020-03-24 16:04 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/indxpath.c | 41 ++++++++ src/backend/optimizer/path/pathkeys.c | 71 +++++++++++-- src/backend/optimizer/path/uniquekey.c | 136 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 37 ++++++- src/backend/optimizer/util/pathnode.c | 46 +++++++-- src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 19 ++++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 2 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 408 insertions(+), 23 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d76fae44b8..16083e7a7e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1723,6 +1723,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2205,6 +2206,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2214,6 +2216,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2401,6 +2404,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4092,6 +4103,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d8..d286b34544 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS = \ joinpath.o \ joinrels.o \ pathkeys.o \ - tidpath.o + tidpath.o \ + uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8286d9cf34..bbc13e6141 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 2a50272da6..363f5349f1 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys = NIL; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + uniquekeys = lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 71b9d42c99..054df9a617 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Checks if the new pathkey's equivalence class is the same as that of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c new file mode 100644 index 0000000000..c421401d0f --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,136 @@ +/*------------------------------------------------------------------------- + * + * uniquekey.c + * Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceClass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ + List *result = NIL; + List *sortkeys; + ListCell *l; + + sortkeys = make_pathkeys_for_uniquekeys(root, + sortclauses, + root->processed_tlist); + + /* Create a uniquekey and add it to the list */ + foreach(l, sortkeys) + { + PathKey *pathkey = (PathKey *) lfirst(l); + EquivalenceClass *ec = pathkey->pk_eclass; + UniqueKey *unique_key = make_canonical_uniquekey(root, ec); + + result = lappend(result, unique_key); + } + + return result; +} + +/* + * uniquekeys_contained_in + * Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ + ListCell *key1, + *key2; + + foreach(key2, keys2) + { + bool found = false; + UniqueKey *uniquekey2 = (UniqueKey *) lfirst(key2); + + foreach(key1, keys1) + { + UniqueKey *uniquekey1 = (UniqueKey *) lfirst(key1); + + if (uniquekey1->eq_clause == uniquekey2->eq_clause) + return true; + } + + if (!found) + return false; + } + + return true; +} + +/* + * has_useful_uniquekeys + * Detect whether the planner could have any uniquekeys that are + * useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ + if (root->query_uniquekeys != NIL) + return true; /* there are some */ + return false; /* definitely useless */ +} + +/* + * make_canonical_uniquekey + * Given the parameters for a UniqueKey, find any pre-existing matching + * uniquekey in the query's list of "canonical" uniquekeys. Make a new + * entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instead, + * equivclass.c will complain if a merge occurs after root->canon_uniquekeys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, + EquivalenceClass *eclass) +{ + UniqueKey *uk; + ListCell *lc; + MemoryContext oldcontext; + + /* The passed eclass might be non-canonical, so chase up to the top */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + foreach(lc, root->canon_uniquekeys) + { + uk = (UniqueKey *) lfirst(lc); + if (eclass == uk->eq_clause) + return uk; + } + + /* + * Be sure canonical uniquekeys are allocated in the main planning context. + * Not an issue in normal planning, but it is for GEQO. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + + uk = makeNode(UniqueKey); + uk->eq_clause = eclass; + + root->canon_uniquekeys = lappend(root->canon_uniquekeys, uk); + + MemoryContextSwitchTo(oldcontext); + + return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 8634940efc..dd64775d8f 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; + root->query_uniquekeys = NIL; } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 62dfc6d44a..3a372af91b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; + root->canon_uniquekeys = NIL; root->left_join_clauses = NIL; root->right_join_clauses = NIL; root->full_join_clauses = NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d6f2153593..a7de8476d9 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3657,15 +3657,30 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6237,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); @@ -7107,6 +7122,26 @@ apply_scanjoin_target_to_paths(PlannerInfo *root, } } + foreach(lc, rel->unique_pathlist) + { + Path *subpath = (Path *) lfirst(lc); + + /* Shouldn't have any parameterized paths anymore */ + Assert(subpath->param_info == NULL); + + if (tlist_same_exprs) + subpath->pathtarget->sortgrouprefs = + scanjoin_target->sortgrouprefs; + else + { + Path *newpath; + + newpath = (Path *) create_projection_path(root, rel, subpath, + scanjoin_target); + lfirst(lc) = newpath; + } + } + /* * Now, if final scan/join target contains SRFs, insert ProjectSetPath(s) * atop each existing path. (Note that this function doesn't look at the diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index e6d08aede5..a4dfafbb59 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -361,9 +361,9 @@ set_cheapest(RelOptInfo *parent_rel) } /* - * add_path + * add_path_to * Consider a potential implementation path for the specified parent rel, - * and add it to the rel's pathlist if it is worthy of consideration. + * and add it to the specified pathlist if it is worthy of consideration. * A path is worthy if it has a better sort order (better pathkeys) or * cheaper cost (on either dimension), or generates fewer rows, than any * existing path that has the same or superset parameterization rels. @@ -416,10 +416,10 @@ set_cheapest(RelOptInfo *parent_rel) * 'parent_rel' is the relation entry to which the path corresponds. * 'new_path' is a potential path for parent_rel. * - * Returns nothing, but modifies parent_rel->pathlist. + * Returns modified pathlist. */ -void -add_path(RelOptInfo *parent_rel, Path *new_path) +static List * +add_path_to(RelOptInfo *parent_rel, List *pathlist, Path *new_path) { bool accept_new = true; /* unless we find a superior old path */ int insert_at = 0; /* where to insert new item */ @@ -440,7 +440,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) * for more than one old path to be tossed out because new_path dominates * it. */ - foreach(p1, parent_rel->pathlist) + foreach(p1, pathlist) { Path *old_path = (Path *) lfirst(p1); bool remove_old = false; /* unless new proves superior */ @@ -584,8 +584,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) */ if (remove_old) { - parent_rel->pathlist = foreach_delete_current(parent_rel->pathlist, - p1); + pathlist = foreach_delete_current(pathlist, p1); /* * Delete the data pointed-to by the deleted cell, if possible @@ -612,8 +611,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) if (accept_new) { /* Accept the new path: insert it at proper place in pathlist */ - parent_rel->pathlist = - list_insert_nth(parent_rel->pathlist, insert_at, new_path); + pathlist = list_insert_nth(pathlist, insert_at, new_path); } else { @@ -621,6 +619,15 @@ add_path(RelOptInfo *parent_rel, Path *new_path) if (!IsA(new_path, IndexPath)) pfree(new_path); } + + return pathlist; +} + +void +add_path(RelOptInfo *parent_rel, Path *new_path) +{ + parent_rel->pathlist = add_path_to(parent_rel, + parent_rel->pathlist, new_path); } /* @@ -915,6 +922,13 @@ add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, return true; } +void +add_unique_path(RelOptInfo *parent_rel, Path *new_path) +{ + parent_rel->unique_pathlist = add_path_to(parent_rel, + parent_rel->unique_pathlist, + new_path); +} /***************************************************************************** * PATH NODE CREATION ROUTINES @@ -940,6 +954,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -964,6 +979,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1000,6 +1016,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1018,6 +1035,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1061,6 +1079,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1922,6 +1941,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1948,6 +1968,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1974,6 +1995,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -1999,6 +2021,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2025,6 +2048,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2051,6 +2075,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2077,6 +2102,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index baced7eec0..a1511b46ea 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3d3be197e0..0de27f0ef3 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* unique keys used for the query */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -657,6 +661,7 @@ typedef struct RelOptInfo List *pathlist; /* Path structures */ List *ppilist; /* ParamPathInfos used in pathlist */ List *partial_pathlist; /* partial Paths */ + List *unique_pathlist; /* unique Paths */ struct Path *cheapest_startup_path; struct Path *cheapest_total_path; struct Path *cheapest_unique_path; @@ -1077,6 +1082,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1120,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1146,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index 6126b491bf..006248bfb5 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e450fe112a..fd25997af5 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -34,6 +34,7 @@ extern void add_partial_path(RelOptInfo *parent_rel, Path *new_path); extern bool add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, List *pathkeys); +extern void add_unique_path(RelOptInfo *parent_rel, Path *new_path); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer, int parallel_workers); extern Path *create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, @@ -44,6 +45,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9ab73bd20c..5b6be383b3 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.0 --y3rsfeqxxd2vddcp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v33-0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH v33 1/2] Unique key @ 2020-03-24 16:04 Dmitrii Dolgov <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: Dmitrii Dolgov @ 2020-03-24 16:04 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 ++++++ src/backend/nodes/print.c | 39 +++++++++++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 +++ src/backend/optimizer/path/indxpath.c | 41 ++++++++++++++++ src/backend/optimizer/path/pathkeys.c | 71 ++++++++++++++++++++++----- src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 37 +++++++++++++- src/backend/optimizer/util/pathnode.c | 46 +++++++++++++---- src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 19 +++++++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 2 + src/include/optimizer/paths.h | 11 +++++ 15 files changed, 272 insertions(+), 23 deletions(-) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d76fae44b8..16083e7a7e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1723,6 +1723,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2205,6 +2206,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2214,6 +2216,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2401,6 +2404,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4092,6 +4103,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d8..d286b34544 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS = \ joinpath.o \ joinrels.o \ pathkeys.o \ - tidpath.o + tidpath.o \ + uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8286d9cf34..bbc13e6141 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 2a50272da6..363f5349f1 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys = NIL; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + uniquekeys = lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 71b9d42c99..054df9a617 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Checks if the new pathkey's equivalence class is the same as that of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 8634940efc..dd64775d8f 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; + root->query_uniquekeys = NIL; } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 62dfc6d44a..3a372af91b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; + root->canon_uniquekeys = NIL; root->left_join_clauses = NIL; root->right_join_clauses = NIL; root->full_join_clauses = NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d6f2153593..a7de8476d9 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3657,15 +3657,30 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6237,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); @@ -7107,6 +7122,26 @@ apply_scanjoin_target_to_paths(PlannerInfo *root, } } + foreach(lc, rel->unique_pathlist) + { + Path *subpath = (Path *) lfirst(lc); + + /* Shouldn't have any parameterized paths anymore */ + Assert(subpath->param_info == NULL); + + if (tlist_same_exprs) + subpath->pathtarget->sortgrouprefs = + scanjoin_target->sortgrouprefs; + else + { + Path *newpath; + + newpath = (Path *) create_projection_path(root, rel, subpath, + scanjoin_target); + lfirst(lc) = newpath; + } + } + /* * Now, if final scan/join target contains SRFs, insert ProjectSetPath(s) * atop each existing path. (Note that this function doesn't look at the diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index e6d08aede5..a4dfafbb59 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -361,9 +361,9 @@ set_cheapest(RelOptInfo *parent_rel) } /* - * add_path + * add_path_to * Consider a potential implementation path for the specified parent rel, - * and add it to the rel's pathlist if it is worthy of consideration. + * and add it to the specified pathlist if it is worthy of consideration. * A path is worthy if it has a better sort order (better pathkeys) or * cheaper cost (on either dimension), or generates fewer rows, than any * existing path that has the same or superset parameterization rels. @@ -416,10 +416,10 @@ set_cheapest(RelOptInfo *parent_rel) * 'parent_rel' is the relation entry to which the path corresponds. * 'new_path' is a potential path for parent_rel. * - * Returns nothing, but modifies parent_rel->pathlist. + * Returns modified pathlist. */ -void -add_path(RelOptInfo *parent_rel, Path *new_path) +static List * +add_path_to(RelOptInfo *parent_rel, List *pathlist, Path *new_path) { bool accept_new = true; /* unless we find a superior old path */ int insert_at = 0; /* where to insert new item */ @@ -440,7 +440,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) * for more than one old path to be tossed out because new_path dominates * it. */ - foreach(p1, parent_rel->pathlist) + foreach(p1, pathlist) { Path *old_path = (Path *) lfirst(p1); bool remove_old = false; /* unless new proves superior */ @@ -584,8 +584,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) */ if (remove_old) { - parent_rel->pathlist = foreach_delete_current(parent_rel->pathlist, - p1); + pathlist = foreach_delete_current(pathlist, p1); /* * Delete the data pointed-to by the deleted cell, if possible @@ -612,8 +611,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) if (accept_new) { /* Accept the new path: insert it at proper place in pathlist */ - parent_rel->pathlist = - list_insert_nth(parent_rel->pathlist, insert_at, new_path); + pathlist = list_insert_nth(pathlist, insert_at, new_path); } else { @@ -621,6 +619,15 @@ add_path(RelOptInfo *parent_rel, Path *new_path) if (!IsA(new_path, IndexPath)) pfree(new_path); } + + return pathlist; +} + +void +add_path(RelOptInfo *parent_rel, Path *new_path) +{ + parent_rel->pathlist = add_path_to(parent_rel, + parent_rel->pathlist, new_path); } /* @@ -915,6 +922,13 @@ add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, return true; } +void +add_unique_path(RelOptInfo *parent_rel, Path *new_path) +{ + parent_rel->unique_pathlist = add_path_to(parent_rel, + parent_rel->unique_pathlist, + new_path); +} /***************************************************************************** * PATH NODE CREATION ROUTINES @@ -940,6 +954,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -964,6 +979,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1000,6 +1016,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1018,6 +1035,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1061,6 +1079,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1922,6 +1941,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1948,6 +1968,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1974,6 +1995,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -1999,6 +2021,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2025,6 +2048,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2051,6 +2075,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2077,6 +2102,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index baced7eec0..a1511b46ea 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3d3be197e0..0de27f0ef3 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* unique keys used for the query */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -657,6 +661,7 @@ typedef struct RelOptInfo List *pathlist; /* Path structures */ List *ppilist; /* ParamPathInfos used in pathlist */ List *partial_pathlist; /* partial Paths */ + List *unique_pathlist; /* unique Paths */ struct Path *cheapest_startup_path; struct Path *cheapest_total_path; struct Path *cheapest_unique_path; @@ -1077,6 +1082,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1120,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1146,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index 6126b491bf..006248bfb5 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e450fe112a..fd25997af5 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -34,6 +34,7 @@ extern void add_partial_path(RelOptInfo *parent_rel, Path *new_path); extern bool add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, List *pathkeys); +extern void add_unique_path(RelOptInfo *parent_rel, Path *new_path); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer, int parallel_workers); extern Path *create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, @@ -44,6 +45,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9ab73bd20c..5b6be383b3 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.0 --qfje4fcsuvt2suqm Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v33-0002-Index-skip-scan.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH v33 1/2] Unique key @ 2020-03-24 16:04 Dmitrii Dolgov <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: Dmitrii Dolgov @ 2020-03-24 16:04 UTC (permalink / raw) Design by David Rowley. Author: Jesper Pedersen --- src/backend/nodes/outfuncs.c | 14 +++ src/backend/nodes/print.c | 39 +++++++ src/backend/optimizer/path/Makefile | 3 +- src/backend/optimizer/path/allpaths.c | 8 ++ src/backend/optimizer/path/indxpath.c | 41 ++++++++ src/backend/optimizer/path/pathkeys.c | 71 +++++++++++-- src/backend/optimizer/path/uniquekey.c | 136 +++++++++++++++++++++++++ src/backend/optimizer/plan/planagg.c | 1 + src/backend/optimizer/plan/planmain.c | 1 + src/backend/optimizer/plan/planner.c | 37 ++++++- src/backend/optimizer/util/pathnode.c | 46 +++++++-- src/include/nodes/nodes.h | 1 + src/include/nodes/pathnodes.h | 19 ++++ src/include/nodes/print.h | 1 + src/include/optimizer/pathnode.h | 2 + src/include/optimizer/paths.h | 11 ++ 16 files changed, 408 insertions(+), 23 deletions(-) create mode 100644 src/backend/optimizer/path/uniquekey.c diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index d76fae44b8..16083e7a7e 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1723,6 +1723,7 @@ _outPathInfo(StringInfo str, const Path *node) WRITE_FLOAT_FIELD(startup_cost, "%.2f"); WRITE_FLOAT_FIELD(total_cost, "%.2f"); WRITE_NODE_FIELD(pathkeys); + WRITE_NODE_FIELD(uniquekeys); } /* @@ -2205,6 +2206,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(eq_classes); WRITE_BOOL_FIELD(ec_merging_done); WRITE_NODE_FIELD(canon_pathkeys); + WRITE_NODE_FIELD(canon_uniquekeys); WRITE_NODE_FIELD(left_join_clauses); WRITE_NODE_FIELD(right_join_clauses); WRITE_NODE_FIELD(full_join_clauses); @@ -2214,6 +2216,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(fkey_list); WRITE_NODE_FIELD(query_pathkeys); + WRITE_NODE_FIELD(query_uniquekeys); WRITE_NODE_FIELD(group_pathkeys); WRITE_NODE_FIELD(window_pathkeys); WRITE_NODE_FIELD(distinct_pathkeys); @@ -2401,6 +2404,14 @@ _outPathKey(StringInfo str, const PathKey *node) WRITE_BOOL_FIELD(pk_nulls_first); } +static void +_outUniqueKey(StringInfo str, const UniqueKey *node) +{ + WRITE_NODE_TYPE("UNIQUEKEY"); + + WRITE_NODE_FIELD(eq_clause); +} + static void _outPathTarget(StringInfo str, const PathTarget *node) { @@ -4092,6 +4103,9 @@ outNode(StringInfo str, const void *obj) case T_PathKey: _outPathKey(str, obj); break; + case T_UniqueKey: + _outUniqueKey(str, obj); + break; case T_PathTarget: _outPathTarget(str, obj); break; diff --git a/src/backend/nodes/print.c b/src/backend/nodes/print.c index 42476724d8..d286b34544 100644 --- a/src/backend/nodes/print.c +++ b/src/backend/nodes/print.c @@ -459,6 +459,45 @@ print_pathkeys(const List *pathkeys, const List *rtable) printf(")\n"); } +/* + * print_uniquekeys - + * uniquekeys list of UniqueKeys + */ +void +print_uniquekeys(const List *uniquekeys, const List *rtable) +{ + ListCell *l; + + printf("("); + foreach(l, uniquekeys) + { + UniqueKey *unique_key = (UniqueKey *) lfirst(l); + EquivalenceClass *eclass = (EquivalenceClass *) unique_key->eq_clause; + ListCell *k; + bool first = true; + + /* chase up */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + printf("("); + foreach(k, eclass->ec_members) + { + EquivalenceMember *mem = (EquivalenceMember *) lfirst(k); + + if (first) + first = false; + else + printf(", "); + print_expr((Node *) mem->em_expr, rtable); + } + printf(")"); + if (lnext(uniquekeys, l)) + printf(", "); + } + printf(")\n"); +} + /* * print_tl * print targetlist in a more legible way. diff --git a/src/backend/optimizer/path/Makefile b/src/backend/optimizer/path/Makefile index 1e199ff66f..63cc1505d9 100644 --- a/src/backend/optimizer/path/Makefile +++ b/src/backend/optimizer/path/Makefile @@ -21,6 +21,7 @@ OBJS = \ joinpath.o \ joinrels.o \ pathkeys.o \ - tidpath.o + tidpath.o \ + uniquekey.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 8286d9cf34..bbc13e6141 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -3954,6 +3954,14 @@ print_path(PlannerInfo *root, Path *path, int indent) print_pathkeys(path->pathkeys, root->parse->rtable); } + if (path->uniquekeys) + { + for (i = 0; i < indent; i++) + printf("\t"); + printf(" uniquekeys: "); + print_uniquekeys(path->uniquekeys, root->parse->rtable); + } + if (join) { JoinPath *jp = (JoinPath *) path; diff --git a/src/backend/optimizer/path/indxpath.c b/src/backend/optimizer/path/indxpath.c index 2a50272da6..363f5349f1 100644 --- a/src/backend/optimizer/path/indxpath.c +++ b/src/backend/optimizer/path/indxpath.c @@ -189,6 +189,7 @@ static Expr *match_clause_to_ordering_op(IndexOptInfo *index, static bool ec_member_matches_indexcol(PlannerInfo *root, RelOptInfo *rel, EquivalenceClass *ec, EquivalenceMember *em, void *arg); +static List *get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys); /* @@ -874,6 +875,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, List *orderbyclausecols; List *index_pathkeys; List *useful_pathkeys; + List *useful_uniquekeys = NIL; bool found_lower_saop_clause; bool pathkeys_possibly_useful; bool index_is_ordered; @@ -1036,11 +1038,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, if (index_clauses != NIL || useful_pathkeys != NIL || useful_predicate || index_only_scan) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1063,6 +1069,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, orderbyclauses, orderbyclausecols, useful_pathkeys, + useful_uniquekeys, index_is_ordered ? ForwardScanDirection : NoMovementScanDirection, @@ -1093,11 +1100,15 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, index_pathkeys); if (useful_pathkeys != NIL) { + if (has_useful_uniquekeys(root)) + useful_uniquekeys = get_uniquekeys_for_index(root, useful_pathkeys); + ipath = create_index_path(root, index, index_clauses, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -1115,6 +1126,7 @@ build_index_paths(PlannerInfo *root, RelOptInfo *rel, NIL, NIL, useful_pathkeys, + useful_uniquekeys, BackwardScanDirection, index_only_scan, outer_relids, @@ -3365,6 +3377,35 @@ match_clause_to_ordering_op(IndexOptInfo *index, return clause; } +/* + * get_uniquekeys_for_index + */ +static List * +get_uniquekeys_for_index(PlannerInfo *root, List *pathkeys) +{ + ListCell *lc; + + if (pathkeys) + { + List *uniquekeys = NIL; + foreach(lc, pathkeys) + { + UniqueKey *unique_key; + PathKey *pk = (PathKey *) lfirst(lc); + EquivalenceClass *ec = (EquivalenceClass *) pk->pk_eclass; + + unique_key = makeNode(UniqueKey); + unique_key->eq_clause = ec; + + uniquekeys = lappend(uniquekeys, unique_key); + } + + if (uniquekeys_contained_in(root->canon_uniquekeys, uniquekeys)) + return uniquekeys; + } + + return NIL; +} /**************************************************************************** * ---- ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS ---- diff --git a/src/backend/optimizer/path/pathkeys.c b/src/backend/optimizer/path/pathkeys.c index 71b9d42c99..054df9a617 100644 --- a/src/backend/optimizer/path/pathkeys.c +++ b/src/backend/optimizer/path/pathkeys.c @@ -29,6 +29,7 @@ #include "utils/lsyscache.h" +static bool pathkey_is_unique(PathKey *new_pathkey, List *pathkeys); static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys); static bool matches_boolean_partition_clause(RestrictInfo *rinfo, RelOptInfo *partrel, @@ -96,6 +97,29 @@ make_canonical_pathkey(PlannerInfo *root, return pk; } +/* + * pathkey_is_unique + * Checks if the new pathkey's equivalence class is the same as that of + * any existing member of the pathkey list. + */ +static bool +pathkey_is_unique(PathKey *new_pathkey, List *pathkeys) +{ + EquivalenceClass *new_ec = new_pathkey->pk_eclass; + ListCell *lc; + + /* If same EC already is already in the list, then not unique */ + foreach(lc, pathkeys) + { + PathKey *old_pathkey = (PathKey *) lfirst(lc); + + if (new_ec == old_pathkey->pk_eclass) + return false; + } + + return true; +} + /* * pathkey_is_redundant * Is a pathkey redundant with one already in the given list? @@ -135,22 +159,12 @@ static bool pathkey_is_redundant(PathKey *new_pathkey, List *pathkeys) { EquivalenceClass *new_ec = new_pathkey->pk_eclass; - ListCell *lc; /* Check for EC containing a constant --- unconditionally redundant */ if (EC_MUST_BE_REDUNDANT(new_ec)) return true; - /* If same EC already used in list, then redundant */ - foreach(lc, pathkeys) - { - PathKey *old_pathkey = (PathKey *) lfirst(lc); - - if (new_ec == old_pathkey->pk_eclass) - return true; - } - - return false; + return !pathkey_is_unique(new_pathkey, pathkeys); } /* @@ -1098,6 +1112,41 @@ make_pathkeys_for_sortclauses(PlannerInfo *root, return pathkeys; } +/* + * make_pathkeys_for_uniquekeyclauses + * Generate a pathkeys list to be used for uniquekey clauses + */ +List * +make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist) +{ + List *pathkeys = NIL; + ListCell *l; + + foreach(l, sortclauses) + { + SortGroupClause *sortcl = (SortGroupClause *) lfirst(l); + Expr *sortkey; + PathKey *pathkey; + + sortkey = (Expr *) get_sortgroupclause_expr(sortcl, tlist); + Assert(OidIsValid(sortcl->sortop)); + pathkey = make_pathkey_from_sortop(root, + sortkey, + root->nullable_baserels, + sortcl->sortop, + sortcl->nulls_first, + sortcl->tleSortGroupRef, + true); + + if (pathkey_is_unique(pathkey, pathkeys)) + pathkeys = lappend(pathkeys, pathkey); + } + + return pathkeys; +} + /**************************************************************************** * PATHKEYS AND MERGECLAUSES ****************************************************************************/ diff --git a/src/backend/optimizer/path/uniquekey.c b/src/backend/optimizer/path/uniquekey.c new file mode 100644 index 0000000000..c421401d0f --- /dev/null +++ b/src/backend/optimizer/path/uniquekey.c @@ -0,0 +1,136 @@ +/*------------------------------------------------------------------------- + * + * uniquekey.c + * Utilities for matching and building unique keys + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/optimizer/path/uniquekey.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "nodes/pg_list.h" + +static UniqueKey *make_canonical_uniquekey(PlannerInfo *root, EquivalenceClass *eclass); + +/* + * Build a list of unique keys + */ +List* +build_uniquekeys(PlannerInfo *root, List *sortclauses) +{ + List *result = NIL; + List *sortkeys; + ListCell *l; + + sortkeys = make_pathkeys_for_uniquekeys(root, + sortclauses, + root->processed_tlist); + + /* Create a uniquekey and add it to the list */ + foreach(l, sortkeys) + { + PathKey *pathkey = (PathKey *) lfirst(l); + EquivalenceClass *ec = pathkey->pk_eclass; + UniqueKey *unique_key = make_canonical_uniquekey(root, ec); + + result = lappend(result, unique_key); + } + + return result; +} + +/* + * uniquekeys_contained_in + * Are the keys2 included in the keys1 superset + */ +bool +uniquekeys_contained_in(List *keys1, List *keys2) +{ + ListCell *key1, + *key2; + + foreach(key2, keys2) + { + bool found = false; + UniqueKey *uniquekey2 = (UniqueKey *) lfirst(key2); + + foreach(key1, keys1) + { + UniqueKey *uniquekey1 = (UniqueKey *) lfirst(key1); + + if (uniquekey1->eq_clause == uniquekey2->eq_clause) + return true; + } + + if (!found) + return false; + } + + return true; +} + +/* + * has_useful_uniquekeys + * Detect whether the planner could have any uniquekeys that are + * useful. + */ +bool +has_useful_uniquekeys(PlannerInfo *root) +{ + if (root->query_uniquekeys != NIL) + return true; /* there are some */ + return false; /* definitely useless */ +} + +/* + * make_canonical_uniquekey + * Given the parameters for a UniqueKey, find any pre-existing matching + * uniquekey in the query's list of "canonical" uniquekeys. Make a new + * entry if there's not one already. + * + * Note that this function must not be used until after we have completed + * merging EquivalenceClasses. (We don't try to enforce that here; instead, + * equivclass.c will complain if a merge occurs after root->canon_uniquekeys + * has become nonempty.) + */ +static UniqueKey * +make_canonical_uniquekey(PlannerInfo *root, + EquivalenceClass *eclass) +{ + UniqueKey *uk; + ListCell *lc; + MemoryContext oldcontext; + + /* The passed eclass might be non-canonical, so chase up to the top */ + while (eclass->ec_merged) + eclass = eclass->ec_merged; + + foreach(lc, root->canon_uniquekeys) + { + uk = (UniqueKey *) lfirst(lc); + if (eclass == uk->eq_clause) + return uk; + } + + /* + * Be sure canonical uniquekeys are allocated in the main planning context. + * Not an issue in normal planning, but it is for GEQO. + */ + oldcontext = MemoryContextSwitchTo(root->planner_cxt); + + uk = makeNode(UniqueKey); + uk->eq_clause = eclass; + + root->canon_uniquekeys = lappend(root->canon_uniquekeys, uk); + + MemoryContextSwitchTo(oldcontext); + + return uk; +} diff --git a/src/backend/optimizer/plan/planagg.c b/src/backend/optimizer/plan/planagg.c index 8634940efc..dd64775d8f 100644 --- a/src/backend/optimizer/plan/planagg.c +++ b/src/backend/optimizer/plan/planagg.c @@ -511,6 +511,7 @@ minmax_qp_callback(PlannerInfo *root, void *extra) root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; + root->query_uniquekeys = NIL; } /* diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 62dfc6d44a..3a372af91b 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -70,6 +70,7 @@ query_planner(PlannerInfo *root, root->join_rel_level = NULL; root->join_cur_level = 0; root->canon_pathkeys = NIL; + root->canon_uniquekeys = NIL; root->left_join_clauses = NIL; root->right_join_clauses = NIL; root->full_join_clauses = NIL; diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index d6f2153593..a7de8476d9 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -3657,15 +3657,30 @@ standard_qp_callback(PlannerInfo *root, void *extra) * much easier, since we know that the parser ensured that one is a * superset of the other. */ + root->query_uniquekeys = NIL; + if (root->group_pathkeys) + { root->query_pathkeys = root->group_pathkeys; + + if (!root->parse->hasAggs) + root->query_uniquekeys = build_uniquekeys(root, qp_extra->groupClause); + } else if (root->window_pathkeys) root->query_pathkeys = root->window_pathkeys; else if (list_length(root->distinct_pathkeys) > list_length(root->sort_pathkeys)) + { root->query_pathkeys = root->distinct_pathkeys; + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else if (root->sort_pathkeys) + { root->query_pathkeys = root->sort_pathkeys; + + if (root->distinct_pathkeys) + root->query_uniquekeys = build_uniquekeys(root, parse->distinctClause); + } else root->query_pathkeys = NIL; } @@ -6222,7 +6237,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) /* Estimate the cost of index scan */ indexScanPath = create_index_path(root, indexInfo, - NIL, NIL, NIL, NIL, + NIL, NIL, NIL, NIL, NIL, ForwardScanDirection, false, NULL, 1.0, false); @@ -7107,6 +7122,26 @@ apply_scanjoin_target_to_paths(PlannerInfo *root, } } + foreach(lc, rel->unique_pathlist) + { + Path *subpath = (Path *) lfirst(lc); + + /* Shouldn't have any parameterized paths anymore */ + Assert(subpath->param_info == NULL); + + if (tlist_same_exprs) + subpath->pathtarget->sortgrouprefs = + scanjoin_target->sortgrouprefs; + else + { + Path *newpath; + + newpath = (Path *) create_projection_path(root, rel, subpath, + scanjoin_target); + lfirst(lc) = newpath; + } + } + /* * Now, if final scan/join target contains SRFs, insert ProjectSetPath(s) * atop each existing path. (Note that this function doesn't look at the diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index e6d08aede5..a4dfafbb59 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -361,9 +361,9 @@ set_cheapest(RelOptInfo *parent_rel) } /* - * add_path + * add_path_to * Consider a potential implementation path for the specified parent rel, - * and add it to the rel's pathlist if it is worthy of consideration. + * and add it to the specified pathlist if it is worthy of consideration. * A path is worthy if it has a better sort order (better pathkeys) or * cheaper cost (on either dimension), or generates fewer rows, than any * existing path that has the same or superset parameterization rels. @@ -416,10 +416,10 @@ set_cheapest(RelOptInfo *parent_rel) * 'parent_rel' is the relation entry to which the path corresponds. * 'new_path' is a potential path for parent_rel. * - * Returns nothing, but modifies parent_rel->pathlist. + * Returns modified pathlist. */ -void -add_path(RelOptInfo *parent_rel, Path *new_path) +static List * +add_path_to(RelOptInfo *parent_rel, List *pathlist, Path *new_path) { bool accept_new = true; /* unless we find a superior old path */ int insert_at = 0; /* where to insert new item */ @@ -440,7 +440,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) * for more than one old path to be tossed out because new_path dominates * it. */ - foreach(p1, parent_rel->pathlist) + foreach(p1, pathlist) { Path *old_path = (Path *) lfirst(p1); bool remove_old = false; /* unless new proves superior */ @@ -584,8 +584,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) */ if (remove_old) { - parent_rel->pathlist = foreach_delete_current(parent_rel->pathlist, - p1); + pathlist = foreach_delete_current(pathlist, p1); /* * Delete the data pointed-to by the deleted cell, if possible @@ -612,8 +611,7 @@ add_path(RelOptInfo *parent_rel, Path *new_path) if (accept_new) { /* Accept the new path: insert it at proper place in pathlist */ - parent_rel->pathlist = - list_insert_nth(parent_rel->pathlist, insert_at, new_path); + pathlist = list_insert_nth(pathlist, insert_at, new_path); } else { @@ -621,6 +619,15 @@ add_path(RelOptInfo *parent_rel, Path *new_path) if (!IsA(new_path, IndexPath)) pfree(new_path); } + + return pathlist; +} + +void +add_path(RelOptInfo *parent_rel, Path *new_path) +{ + parent_rel->pathlist = add_path_to(parent_rel, + parent_rel->pathlist, new_path); } /* @@ -915,6 +922,13 @@ add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, return true; } +void +add_unique_path(RelOptInfo *parent_rel, Path *new_path) +{ + parent_rel->unique_pathlist = add_path_to(parent_rel, + parent_rel->unique_pathlist, + new_path); +} /***************************************************************************** * PATH NODE CREATION ROUTINES @@ -940,6 +954,7 @@ create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = parallel_workers; pathnode->pathkeys = NIL; /* seqscan has unordered result */ + pathnode->uniquekeys = NIL; cost_seqscan(pathnode, root, rel, pathnode->param_info); @@ -964,6 +979,7 @@ create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* samplescan has unordered result */ + pathnode->uniquekeys = NIL; cost_samplescan(pathnode, root, rel, pathnode->param_info); @@ -1000,6 +1016,7 @@ create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, @@ -1018,6 +1035,7 @@ create_index_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = 0; pathnode->path.pathkeys = pathkeys; + pathnode->path.uniquekeys = uniquekeys; pathnode->indexinfo = index; pathnode->indexclauses = indexclauses; @@ -1061,6 +1079,7 @@ create_bitmap_heap_path(PlannerInfo *root, pathnode->path.parallel_safe = rel->consider_parallel; pathnode->path.parallel_workers = parallel_degree; pathnode->path.pathkeys = NIL; /* always unordered */ + pathnode->path.uniquekeys = NIL; pathnode->bitmapqual = bitmapqual; @@ -1922,6 +1941,7 @@ create_functionscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = pathkeys; + pathnode->uniquekeys = NIL; cost_functionscan(pathnode, root, rel, pathnode->param_info); @@ -1948,6 +1968,7 @@ create_tablefuncscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_tablefuncscan(pathnode, root, rel, pathnode->param_info); @@ -1974,6 +1995,7 @@ create_valuesscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_valuesscan(pathnode, root, rel, pathnode->param_info); @@ -1999,6 +2021,7 @@ create_ctescan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer) pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* XXX for now, result is always unordered */ + pathnode->uniquekeys = NIL; cost_ctescan(pathnode, root, rel, pathnode->param_info); @@ -2025,6 +2048,7 @@ create_namedtuplestorescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_namedtuplestorescan(pathnode, root, rel, pathnode->param_info); @@ -2051,6 +2075,7 @@ create_resultscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; cost_resultscan(pathnode, root, rel, pathnode->param_info); @@ -2077,6 +2102,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->parallel_safe = rel->consider_parallel; pathnode->parallel_workers = 0; pathnode->pathkeys = NIL; /* result is always unordered */ + pathnode->uniquekeys = NIL; /* Cost is the same as for a regular CTE scan */ cost_ctescan(pathnode, root, rel, pathnode->param_info); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index baced7eec0..a1511b46ea 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -261,6 +261,7 @@ typedef enum NodeTag T_EquivalenceMember, T_PathKey, T_PathTarget, + T_UniqueKey, T_RestrictInfo, T_IndexClause, T_PlaceHolderVar, diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 3d3be197e0..0de27f0ef3 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -269,6 +269,8 @@ struct PlannerInfo List *canon_pathkeys; /* list of "canonical" PathKeys */ + List *canon_uniquekeys; /* list of "canonical" UniqueKeys */ + List *left_join_clauses; /* list of RestrictInfos for mergejoinable * outer join clauses w/nonnullable var on * left */ @@ -297,6 +299,8 @@ struct PlannerInfo List *query_pathkeys; /* desired pathkeys for query_planner() */ + List *query_uniquekeys; /* unique keys used for the query */ + List *group_pathkeys; /* groupClause pathkeys, if any */ List *window_pathkeys; /* pathkeys of bottom window, if any */ List *distinct_pathkeys; /* distinctClause pathkeys, if any */ @@ -657,6 +661,7 @@ typedef struct RelOptInfo List *pathlist; /* Path structures */ List *ppilist; /* ParamPathInfos used in pathlist */ List *partial_pathlist; /* partial Paths */ + List *unique_pathlist; /* unique Paths */ struct Path *cheapest_startup_path; struct Path *cheapest_total_path; struct Path *cheapest_unique_path; @@ -1077,6 +1082,15 @@ typedef struct ParamPathInfo List *ppi_clauses; /* join clauses available from outer rels */ } ParamPathInfo; +/* + * UniqueKey + */ +typedef struct UniqueKey +{ + NodeTag type; + + EquivalenceClass *eq_clause; /* equivalence class */ +} UniqueKey; /* * Type "Path" is used as-is for sequential-scan paths, as well as some other @@ -1106,6 +1120,9 @@ typedef struct ParamPathInfo * * "pathkeys" is a List of PathKey nodes (see above), describing the sort * ordering of the path's output rows. + * + * "uniquekeys", if not NIL, is a list of UniqueKey nodes (see above), + * describing the XXX. */ typedef struct Path { @@ -1129,6 +1146,8 @@ typedef struct Path List *pathkeys; /* sort ordering of path's output */ /* pathkeys is a List of PathKey nodes; see above */ + + List *uniquekeys; /* the unique keys, or NIL if none */ } Path; /* Macro for extracting a path's parameterization relids; beware double eval */ diff --git a/src/include/nodes/print.h b/src/include/nodes/print.h index 6126b491bf..006248bfb5 100644 --- a/src/include/nodes/print.h +++ b/src/include/nodes/print.h @@ -28,6 +28,7 @@ extern char *pretty_format_node_dump(const char *dump); extern void print_rt(const List *rtable); extern void print_expr(const Node *expr, const List *rtable); extern void print_pathkeys(const List *pathkeys, const List *rtable); +extern void print_uniquekeys(const List *uniquekeys, const List *rtable); extern void print_tl(const List *tlist, const List *rtable); extern void print_slot(TupleTableSlot *slot); diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index e450fe112a..fd25997af5 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -34,6 +34,7 @@ extern void add_partial_path(RelOptInfo *parent_rel, Path *new_path); extern bool add_partial_path_precheck(RelOptInfo *parent_rel, Cost total_cost, List *pathkeys); +extern void add_unique_path(RelOptInfo *parent_rel, Path *new_path); extern Path *create_seqscan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer, int parallel_workers); extern Path *create_samplescan_path(PlannerInfo *root, RelOptInfo *rel, @@ -44,6 +45,7 @@ extern IndexPath *create_index_path(PlannerInfo *root, List *indexorderbys, List *indexorderbycols, List *pathkeys, + List *uniquekeys, ScanDirection indexscandir, bool indexonly, Relids required_outer, diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h index 9ab73bd20c..5b6be383b3 100644 --- a/src/include/optimizer/paths.h +++ b/src/include/optimizer/paths.h @@ -214,6 +214,9 @@ extern List *build_join_pathkeys(PlannerInfo *root, extern List *make_pathkeys_for_sortclauses(PlannerInfo *root, List *sortclauses, List *tlist); +extern List *make_pathkeys_for_uniquekeys(PlannerInfo *root, + List *sortclauses, + List *tlist); extern void initialize_mergeclause_eclasses(PlannerInfo *root, RestrictInfo *restrictinfo); extern void update_mergeclause_eclasses(PlannerInfo *root, @@ -240,4 +243,12 @@ extern PathKey *make_canonical_pathkey(PlannerInfo *root, extern void add_paths_to_append_rel(PlannerInfo *root, RelOptInfo *rel, List *live_childrels); +/* + * uniquekey.c + * Utilities for matching and building unique keys + */ +extern List *build_uniquekeys(PlannerInfo *root, List *sortclauses); +extern bool uniquekeys_contained_in(List *keys1, List *keys2); +extern bool has_useful_uniquekeys(PlannerInfo *root); + #endif /* PATHS_H */ -- 2.21.0 --t7fko7qta7zyrtxo Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v33-0002-Index-skip-scan-with-filtering.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
* [PATCH v25 1/4] Add REPACK command @ 2025-07-26 17:57 Álvaro Herrera <[email protected]> 0 siblings, 0 replies; 11+ messages in thread From: Álvaro Herrera @ 2025-07-26 17:57 UTC (permalink / raw) REPACK absorbs the functionality of VACUUM FULL and CLUSTER in a single command. Because this functionality is completely different from regular VACUUM, having it separate from VACUUM makes it easier for users to understand; as for CLUSTER, the term is heavily overloaded in the IT world and even in Postgres itself, so it's good that we can avoid it. This also adds pg_repackdb, a new utility that can invoke the new commands. This is heavily based on vacuumdb. Author: Antonin Houska <[email protected]> Co-authored-by: Álvaro Herrera <[email protected]> Reviewed-by: Mihail Nikalayeu <[email protected]> Reviewed-by: Robert Treat <[email protected]> Reviewed-by: Euler Taveira <[email protected]> Reviewed-by: Matheus Alcantara <[email protected]> Reviewed-by: Junwang Zhao <[email protected]> Reviewed-by: jian he <[email protected]> Discussion: https://postgr.es/m/82651.1720540558@antos Discussion: https://postgr.es/m/[email protected] --- doc/src/sgml/monitoring.sgml | 223 +++++- doc/src/sgml/ref/allfiles.sgml | 2 + doc/src/sgml/ref/cluster.sgml | 97 +-- doc/src/sgml/ref/clusterdb.sgml | 5 + doc/src/sgml/ref/pg_repackdb.sgml | 488 +++++++++++++ doc/src/sgml/ref/repack.sgml | 319 +++++++++ doc/src/sgml/ref/vacuum.sgml | 33 +- doc/src/sgml/reference.sgml | 2 + src/backend/access/heap/heapam_handler.c | 32 +- src/backend/catalog/index.c | 2 +- src/backend/catalog/system_views.sql | 29 +- src/backend/commands/cluster.c | 854 +++++++++++++++-------- src/backend/commands/vacuum.c | 6 +- src/backend/parser/gram.y | 97 ++- src/backend/tcop/utility.c | 20 +- src/backend/utils/adt/pgstatfuncs.c | 2 + src/bin/psql/tab-complete.in.c | 33 +- src/bin/scripts/Makefile | 4 +- src/bin/scripts/meson.build | 2 + src/bin/scripts/pg_repackdb.c | 242 +++++++ src/bin/scripts/t/103_repackdb.pl | 47 ++ src/bin/scripts/vacuuming.c | 114 ++- src/bin/scripts/vacuuming.h | 3 + src/include/commands/cluster.h | 8 +- src/include/commands/progress.h | 48 +- src/include/nodes/parsenodes.h | 35 +- src/include/parser/kwlist.h | 1 + src/include/tcop/cmdtaglist.h | 1 + src/include/utils/backend_progress.h | 1 + src/test/regress/expected/cluster.out | 134 +++- src/test/regress/expected/rules.out | 72 +- src/test/regress/sql/cluster.sql | 70 +- src/tools/pgindent/typedefs.list | 2 + 33 files changed, 2483 insertions(+), 545 deletions(-) create mode 100644 doc/src/sgml/ref/pg_repackdb.sgml create mode 100644 doc/src/sgml/ref/repack.sgml create mode 100644 src/bin/scripts/pg_repackdb.c create mode 100644 src/bin/scripts/t/103_repackdb.pl diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f3bf527d5b4..467e081f015 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -405,6 +405,14 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser </entry> </row> + <row> + <entry><structname>pg_stat_progress_repack</structname><indexterm><primary>pg_stat_progress_repack</primary></indexterm></entry> + <entry>One row for each backend running + <command>REPACK</command>, showing current progress. See + <xref linkend="repack-progress-reporting"/>. + </entry> + </row> + <row> <entry><structname>pg_stat_progress_basebackup</structname><indexterm><primary>pg_stat_progress_basebackup</primary></indexterm></entry> <entry>One row for each WAL sender process streaming a base backup, @@ -5575,7 +5583,8 @@ FROM pg_stat_get_backend_idset() AS backendid; certain commands during command execution. Currently, the only commands which support progress reporting are <command>ANALYZE</command>, <command>CLUSTER</command>, - <command>CREATE INDEX</command>, <command>VACUUM</command>, + <command>CREATE INDEX</command>, <command>REPACK</command>, + <command>VACUUM</command>, <command>COPY</command>, and <xref linkend="protocol-replication-base-backup"/> (i.e., replication command that <xref linkend="app-pgbasebackup"/> issues to take @@ -6034,6 +6043,218 @@ FROM pg_stat_get_backend_idset() AS backendid; </table> </sect2> + <sect2 id="repack-progress-reporting"> + <title>REPACK Progress Reporting</title> + + <indexterm> + <primary>pg_stat_progress_repack</primary> + </indexterm> + + <para> + Whenever <command>REPACK</command> is running, + the <structname>pg_stat_progress_repack</structname> view will contain a + row for each backend that is currently running the command. The tables + below describe the information that will be reported and provide + information about how to interpret it. + </para> + + <table id="pg-stat-progress-repack-view" xreflabel="pg_stat_progress_repack"> + <title><structname>pg_stat_progress_repack</structname> View</title> + <tgroup cols="1"> + <thead> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + Column Type + </para> + <para> + Description + </para></entry> + </row> + </thead> + + <tbody> + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>pid</structfield> <type>integer</type> + </para> + <para> + Process ID of backend. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>datid</structfield> <type>oid</type> + </para> + <para> + OID of the database to which this backend is connected. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>datname</structfield> <type>name</type> + </para> + <para> + Name of the database to which this backend is connected. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>relid</structfield> <type>oid</type> + </para> + <para> + OID of the table being repacked. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>phase</structfield> <type>text</type> + </para> + <para> + Current processing phase. See <xref linkend="repack-phases"/>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>repack_index_relid</structfield> <type>oid</type> + </para> + <para> + If the table is being scanned using an index, this is the OID of the + index being used; otherwise, it is zero. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_tuples_scanned</structfield> <type>bigint</type> + </para> + <para> + Number of heap tuples scanned. + This counter only advances when the phase is + <literal>seq scanning heap</literal>, + <literal>index scanning heap</literal> + or <literal>writing new heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_tuples_written</structfield> <type>bigint</type> + </para> + <para> + Number of heap tuples written. + This counter only advances when the phase is + <literal>seq scanning heap</literal>, + <literal>index scanning heap</literal> + or <literal>writing new heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_blks_total</structfield> <type>bigint</type> + </para> + <para> + Total number of heap blocks in the table. This number is reported + as of the beginning of <literal>seq scanning heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>heap_blks_scanned</structfield> <type>bigint</type> + </para> + <para> + Number of heap blocks scanned. This counter only advances when the + phase is <literal>seq scanning heap</literal>. + </para></entry> + </row> + + <row> + <entry role="catalog_table_entry"><para role="column_definition"> + <structfield>index_rebuild_count</structfield> <type>bigint</type> + </para> + <para> + Number of indexes rebuilt. This counter only advances when the phase + is <literal>rebuilding index</literal>. + </para></entry> + </row> + </tbody> + </tgroup> + </table> + + <table id="repack-phases"> + <title>REPACK Phases</title> + <tgroup cols="2"> + <colspec colname="col1" colwidth="1*"/> + <colspec colname="col2" colwidth="2*"/> + <thead> + <row> + <entry>Phase</entry> + <entry>Description</entry> + </row> + </thead> + + <tbody> + <row> + <entry><literal>initializing</literal></entry> + <entry> + The command is preparing to begin scanning the heap. This phase is + expected to be very brief. + </entry> + </row> + <row> + <entry><literal>seq scanning heap</literal></entry> + <entry> + The command is currently scanning the table using a sequential scan. + </entry> + </row> + <row> + <entry><literal>index scanning heap</literal></entry> + <entry> + <command>REPACK</command> is currently scanning the table using an index scan. + </entry> + </row> + <row> + <entry><literal>sorting tuples</literal></entry> + <entry> + <command>REPACK</command> is currently sorting tuples. + </entry> + </row> + <row> + <entry><literal>writing new heap</literal></entry> + <entry> + <command>REPACK</command> is currently writing the new heap. + </entry> + </row> + <row> + <entry><literal>swapping relation files</literal></entry> + <entry> + The command is currently swapping newly-built files into place. + </entry> + </row> + <row> + <entry><literal>rebuilding index</literal></entry> + <entry> + The command is currently rebuilding an index. + </entry> + </row> + <row> + <entry><literal>performing final cleanup</literal></entry> + <entry> + The command is performing final cleanup. When this phase is + completed, <command>REPACK</command> will end. + </entry> + </row> + </tbody> + </tgroup> + </table> + </sect2> + <sect2 id="copy-progress-reporting"> <title>COPY Progress Reporting</title> diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index f5be638867a..eabf92e3536 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -167,6 +167,7 @@ Complete list of usable sgml source files in this directory. <!ENTITY refreshMaterializedView SYSTEM "refresh_materialized_view.sgml"> <!ENTITY reindex SYSTEM "reindex.sgml"> <!ENTITY releaseSavepoint SYSTEM "release_savepoint.sgml"> +<!ENTITY repack SYSTEM "repack.sgml"> <!ENTITY reset SYSTEM "reset.sgml"> <!ENTITY revoke SYSTEM "revoke.sgml"> <!ENTITY rollback SYSTEM "rollback.sgml"> @@ -212,6 +213,7 @@ Complete list of usable sgml source files in this directory. <!ENTITY pgIsready SYSTEM "pg_isready.sgml"> <!ENTITY pgReceivewal SYSTEM "pg_receivewal.sgml"> <!ENTITY pgRecvlogical SYSTEM "pg_recvlogical.sgml"> +<!ENTITY pgRepackdb SYSTEM "pg_repackdb.sgml"> <!ENTITY pgResetwal SYSTEM "pg_resetwal.sgml"> <!ENTITY pgRestore SYSTEM "pg_restore.sgml"> <!ENTITY pgRewind SYSTEM "pg_rewind.sgml"> diff --git a/doc/src/sgml/ref/cluster.sgml b/doc/src/sgml/ref/cluster.sgml index 8811f169ea0..cfcfb65e349 100644 --- a/doc/src/sgml/ref/cluster.sgml +++ b/doc/src/sgml/ref/cluster.sgml @@ -33,51 +33,13 @@ CLUSTER [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <r <title>Description</title> <para> - <command>CLUSTER</command> instructs <productname>PostgreSQL</productname> - to cluster the table specified - by <replaceable class="parameter">table_name</replaceable> - based on the index specified by - <replaceable class="parameter">index_name</replaceable>. The index must - already have been defined on - <replaceable class="parameter">table_name</replaceable>. + The <command>CLUSTER</command> command is equivalent to + <xref linkend="sql-repack"/> with an <literal>USING INDEX</literal> + clause. See there for more details. </para> - <para> - When a table is clustered, it is physically reordered - based on the index information. Clustering is a one-time operation: - when the table is subsequently updated, the changes are - not clustered. That is, no attempt is made to store new or - updated rows according to their index order. (If one wishes, one can - periodically recluster by issuing the command again. Also, setting - the table's <literal>fillfactor</literal> storage parameter to less than - 100% can aid in preserving cluster ordering during updates, since updated - rows are kept on the same page if enough space is available there.) - </para> +<!-- Do we need to describe exactly which options map to what? They seem obvious to me. --> - <para> - When a table is clustered, <productname>PostgreSQL</productname> - remembers which index it was clustered by. The form - <command>CLUSTER <replaceable class="parameter">table_name</replaceable></command> - reclusters the table using the same index as before. You can also - use the <literal>CLUSTER</literal> or <literal>SET WITHOUT CLUSTER</literal> - forms of <link linkend="sql-altertable"><command>ALTER TABLE</command></link> to set the index to be used for - future cluster operations, or to clear any previous setting. - </para> - - <para> - <command>CLUSTER</command> without a - <replaceable class="parameter">table_name</replaceable> reclusters all the - previously-clustered tables in the current database that the calling user - has privileges for. This form of <command>CLUSTER</command> cannot be - executed inside a transaction block. - </para> - - <para> - When a table is being clustered, an <literal>ACCESS - EXCLUSIVE</literal> lock is acquired on it. This prevents any other - database operations (both reads and writes) from operating on the - table until the <command>CLUSTER</command> is finished. - </para> </refsect1> <refsect1> @@ -136,63 +98,12 @@ CLUSTER [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <r on the table. </para> - <para> - In cases where you are accessing single rows randomly - within a table, the actual order of the data in the - table is unimportant. However, if you tend to access some - data more than others, and there is an index that groups - them together, you will benefit from using <command>CLUSTER</command>. - If you are requesting a range of indexed values from a table, or a - single indexed value that has multiple rows that match, - <command>CLUSTER</command> will help because once the index identifies the - table page for the first row that matches, all other rows - that match are probably already on the same table page, - and so you save disk accesses and speed up the query. - </para> - - <para> - <command>CLUSTER</command> can re-sort the table using either an index scan - on the specified index, or (if the index is a b-tree) a sequential - scan followed by sorting. It will attempt to choose the method that - will be faster, based on planner cost parameters and available statistical - information. - </para> - <para> While <command>CLUSTER</command> is running, the <xref linkend="guc-search-path"/> is temporarily changed to <literal>pg_catalog, pg_temp</literal>. </para> - <para> - When an index scan is used, a temporary copy of the table is created that - contains the table data in the index order. Temporary copies of each - index on the table are created as well. Therefore, you need free space on - disk at least equal to the sum of the table size and the index sizes. - </para> - - <para> - When a sequential scan and sort is used, a temporary sort file is - also created, so that the peak temporary space requirement is as much - as double the table size, plus the index sizes. This method is often - faster than the index scan method, but if the disk space requirement is - intolerable, you can disable this choice by temporarily setting <xref - linkend="guc-enable-sort"/> to <literal>off</literal>. - </para> - - <para> - It is advisable to set <xref linkend="guc-maintenance-work-mem"/> to - a reasonably large value (but not more than the amount of RAM you can - dedicate to the <command>CLUSTER</command> operation) before clustering. - </para> - - <para> - Because the planner records statistics about the ordering of - tables, it is advisable to run <link linkend="sql-analyze"><command>ANALYZE</command></link> - on the newly clustered table. - Otherwise, the planner might make poor choices of query plans. - </para> - <para> Because <command>CLUSTER</command> remembers which indexes are clustered, one can cluster the tables one wants clustered manually the first time, diff --git a/doc/src/sgml/ref/clusterdb.sgml b/doc/src/sgml/ref/clusterdb.sgml index 0d2051bf6f1..546c1289c31 100644 --- a/doc/src/sgml/ref/clusterdb.sgml +++ b/doc/src/sgml/ref/clusterdb.sgml @@ -64,6 +64,11 @@ PostgreSQL documentation this utility and via other methods for accessing the server. </para> + <para> + <application>clusterdb</application> has been superceded by + <application>pg_repackdb</application>. + </para> + </refsect1> diff --git a/doc/src/sgml/ref/pg_repackdb.sgml b/doc/src/sgml/ref/pg_repackdb.sgml new file mode 100644 index 00000000000..b313b54ab63 --- /dev/null +++ b/doc/src/sgml/ref/pg_repackdb.sgml @@ -0,0 +1,488 @@ +<!-- +doc/src/sgml/ref/pg_repackdb.sgml +PostgreSQL documentation +--> + +<refentry id="app-pgrepackdb"> + <indexterm zone="app-pgrepackdb"> + <primary>pg_repackdb</primary> + </indexterm> + + <refmeta> + <refentrytitle><application>pg_repackdb</application></refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo>Application</refmiscinfo> + </refmeta> + + <refnamediv> + <refname>pg_repackdb</refname> + <refpurpose>repack and analyze a <productname>PostgreSQL</productname> + database</refpurpose> + </refnamediv> + + <refsynopsisdiv> + <cmdsynopsis> + <command>pg_repackdb</command> + <arg rep="repeat"><replaceable>connection-option</replaceable></arg> + <arg rep="repeat"><replaceable>option</replaceable></arg> + + <arg choice="plain" rep="repeat"> + <arg choice="opt"> + <group choice="plain"> + <arg choice="plain"><option>-t</option></arg> + <arg choice="plain"><option>--table</option></arg> + </group> + <replaceable>table</replaceable> + <arg choice="opt">( <replaceable class="parameter">column</replaceable> [,...] )</arg> + </arg> + </arg> + + <arg choice="opt"> + <group choice="plain"> + <arg choice="plain"><replaceable>dbname</replaceable></arg> + <arg choice="plain"><option>-a</option></arg> + <arg choice="plain"><option>--all</option></arg> + </group> + </arg> + </cmdsynopsis> + + <cmdsynopsis> + <command>pg_repackdb</command> + <arg rep="repeat"><replaceable>connection-option</replaceable></arg> + <arg rep="repeat"><replaceable>option</replaceable></arg> + + <arg choice="plain" rep="repeat"> + <arg choice="opt"> + <group choice="plain"> + <arg choice="plain"><option>-n</option></arg> + <arg choice="plain"><option>--schema</option></arg> + </group> + <replaceable>schema</replaceable> + </arg> + </arg> + + <arg choice="opt"> + <group choice="plain"> + <arg choice="plain"><replaceable>dbname</replaceable></arg> + <arg choice="plain"><option>-a</option></arg> + <arg choice="plain"><option>--all</option></arg> + </group> + </arg> + </cmdsynopsis> + + <cmdsynopsis> + <command>pg_repackdb</command> + <arg rep="repeat"><replaceable>connection-option</replaceable></arg> + <arg rep="repeat"><replaceable>option</replaceable></arg> + + <arg choice="plain" rep="repeat"> + <arg choice="opt"> + <group choice="plain"> + <arg choice="plain"><option>-N</option></arg> + <arg choice="plain"><option>--exclude-schema</option></arg> + </group> + <replaceable>schema</replaceable> + </arg> + </arg> + + <arg choice="opt"> + <group choice="plain"> + <arg choice="plain"><replaceable>dbname</replaceable></arg> + <arg choice="plain"><option>-a</option></arg> + <arg choice="plain"><option>--all</option></arg> + </group> + </arg> + </cmdsynopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <application>pg_repackdb</application> is a utility for repacking a + <productname>PostgreSQL</productname> database. + <application>pg_repackdb</application> will also generate internal + statistics used by the <productname>PostgreSQL</productname> query + optimizer. + </para> + + <para> + <application>pg_repackdb</application> is a wrapper around the SQL + command <link linkend="sql-repack"><command>REPACK</command></link> There + is no effective difference between repacking and analyzing databases via + this utility and via other methods for accessing the server. + </para> + </refsect1> + + <refsect1> + <title>Options</title> + + <para> + <application>pg_repackdb</application> accepts the following command-line arguments: + <variablelist> + <varlistentry> + <term><option>-a</option></term> + <term><option>--all</option></term> + <listitem> + <para> + Repack all databases. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option><optional>-d</optional> <replaceable class="parameter">dbname</replaceable></option></term> + <term><option><optional>--dbname=</optional><replaceable class="parameter">dbname</replaceable></option></term> + <listitem> + <para> + Specifies the name of the database to be repacked or analyzed, + when <option>-a</option>/<option>--all</option> is not used. If this + is not specified, the database name is read from the environment + variable <envar>PGDATABASE</envar>. If that is not set, the user name + specified for the connection is used. + The <replaceable>dbname</replaceable> can be + a <link linkend="libpq-connstring">connection string</link>. If so, + connection string parameters will override any conflicting command + line options. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-e</option></term> + <term><option>--echo</option></term> + <listitem> + <para> + Echo the commands that <application>pg_repackdb</application> + generates and sends to the server. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>--index<optional>=<replaceable class="parameter">index_name</replaceable></optional></option></term> + <listitem> + <para> + Pass the <literal>USING INDEX</literal> clause to <literal>REPACK</literal>, + and optionally the index name to specify. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-j <replaceable class="parameter">njobs</replaceable></option></term> + <term><option>--jobs=<replaceable class="parameter">njobs</replaceable></option></term> + <listitem> + <para> + Execute the repack or analyze commands in parallel by running + <replaceable class="parameter">njobs</replaceable> + commands simultaneously. This option may reduce the processing time + but it also increases the load on the database server. + </para> + <para> + <application>pg_repackdb</application> will open + <replaceable class="parameter">njobs</replaceable> connections to the + database, so make sure your <xref linkend="guc-max-connections"/> + setting is high enough to accommodate all connections. + </para> + <para> + Note that using this mode might cause deadlock failures if certain + system catalogs are processed in parallel. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-n <replaceable class="parameter">schema</replaceable></option></term> + <term><option>--schema=<replaceable class="parameter">schema</replaceable></option></term> + <listitem> + <para> + Repack or analyze all tables in + <replaceable class="parameter">schema</replaceable> only. Multiple + schemas can be repacked by writing multiple <option>-n</option> + switches. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-N <replaceable class="parameter">schema</replaceable></option></term> + <term><option>--exclude-schema=<replaceable class="parameter">schema</replaceable></option></term> + <listitem> + <para> + Do not repack or analyze any tables in + <replaceable class="parameter">schema</replaceable>. Multiple schemas + can be excluded by writing multiple <option>-N</option> switches. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-q</option></term> + <term><option>--quiet</option></term> + <listitem> + <para> + Do not display progress messages. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-t <replaceable class="parameter">table</replaceable> [ (<replaceable class="parameter">column</replaceable> [,...]) ]</option></term> + <term><option>--table=<replaceable class="parameter">table</replaceable> [ (<replaceable class="parameter">column</replaceable> [,...]) ]</option></term> + <listitem> + <para> + Repack or analyze <replaceable class="parameter">table</replaceable> + only. Column names can be specified only in conjunction with + the <option>--analyze</option> option. Multiple tables can be + repacked by writing multiple + <option>-t</option> switches. + </para> + <tip> + <para> + If you specify columns, you probably have to escape the parentheses + from the shell. (See examples below.) + </para> + </tip> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-v</option></term> + <term><option>--verbose</option></term> + <listitem> + <para> + Print detailed information during processing. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-V</option></term> + <term><option>--version</option></term> + <listitem> + <para> + Print the <application>pg_repackdb</application> version and exit. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-z</option></term> + <term><option>--analyze</option></term> + <listitem> + <para> + Also calculate statistics for use by the optimizer. If a column name + list is given, only compute statistics for those columns. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-?</option></term> + <term><option>--help</option></term> + <listitem> + <para> + Show help about <application>pg_repackdb</application> command line + arguments, and exit. + </para> + </listitem> + </varlistentry> + + </variablelist> + </para> + + <para> + <application>pg_repackdb</application> also accepts + the following command-line arguments for connection parameters: + <variablelist> + <varlistentry> + <term><option>-h <replaceable class="parameter">host</replaceable></option></term> + <term><option>--host=<replaceable class="parameter">host</replaceable></option></term> + <listitem> + <para> + Specifies the host name of the machine on which the server + is running. If the value begins with a slash, it is used + as the directory for the Unix domain socket. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-p <replaceable class="parameter">port</replaceable></option></term> + <term><option>--port=<replaceable class="parameter">port</replaceable></option></term> + <listitem> + <para> + Specifies the TCP port or local Unix domain socket file + extension on which the server + is listening for connections. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-U <replaceable class="parameter">username</replaceable></option></term> + <term><option>--username=<replaceable class="parameter">username</replaceable></option></term> + <listitem> + <para> + User name to connect as. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-w</option></term> + <term><option>--no-password</option></term> + <listitem> + <para> + Never issue a password prompt. If the server requires + password authentication and a password is not available by + other means such as a <filename>.pgpass</filename> file, the + connection attempt will fail. This option can be useful in + batch jobs and scripts where no user is present to enter a + password. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>-W</option></term> + <term><option>--password</option></term> + <listitem> + <para> + Force <application>pg_repackdb</application> to prompt for a + password before connecting to a database. + </para> + + <para> + This option is never essential, since + <application>pg_repackdb</application> will automatically prompt + for a password if the server demands password authentication. + However, <application>pg_repackdb</application> will waste a + connection attempt finding out that the server wants a password. + In some cases it is worth typing <option>-W</option> to avoid the extra + connection attempt. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><option>--maintenance-db=<replaceable class="parameter">dbname</replaceable></option></term> + <listitem> + <para> + When the <option>-a</option>/<option>--all</option> is used, connect + to this database to gather the list of databases to repack. + If not specified, the <literal>postgres</literal> database will be used, + or if that does not exist, <literal>template1</literal> will be used. + This can be a <link linkend="libpq-connstring">connection + string</link>. If so, connection string parameters will override any + conflicting command line options. Also, connection string parameters + other than the database name itself will be re-used when connecting + to other databases. + </para> + </listitem> + </varlistentry> + </variablelist> + </para> + </refsect1> + + + <refsect1> + <title>Environment</title> + + <variablelist> + <varlistentry> + <term><envar>PGDATABASE</envar></term> + <term><envar>PGHOST</envar></term> + <term><envar>PGPORT</envar></term> + <term><envar>PGUSER</envar></term> + + <listitem> + <para> + Default connection parameters + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><envar>PG_COLOR</envar></term> + <listitem> + <para> + Specifies whether to use color in diagnostic messages. Possible values + are <literal>always</literal>, <literal>auto</literal> and + <literal>never</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <para> + This utility, like most other <productname>PostgreSQL</productname> utilities, + also uses the environment variables supported by <application>libpq</application> + (see <xref linkend="libpq-envars"/>). + </para> + + </refsect1> + + + <refsect1> + <title>Diagnostics</title> + + <para> + In case of difficulty, see + <xref linkend="sql-repack"/> and <xref linkend="app-psql"/> for + discussions of potential problems and error messages. + The database server must be running at the + targeted host. Also, any default connection settings and environment + variables used by the <application>libpq</application> front-end + library will apply. + </para> + + </refsect1> + + <refsect1> + <title>Examples</title> + + <para> + To repack the database <literal>test</literal>: +<screen> +<prompt>$ </prompt><userinput>pg_repackdb test</userinput> +</screen> + </para> + + <para> + To repack and analyze for the optimizer a database named + <literal>bigdb</literal>: +<screen> +<prompt>$ </prompt><userinput>pg_repackdb --analyze bigdb</userinput> +</screen> + </para> + + <para> + To repack a single table + <literal>foo</literal> in a database named + <literal>xyzzy</literal>, and analyze a single column + <literal>bar</literal> of the table for the optimizer: +<screen> +<prompt>$ </prompt><userinput>pg_repackdb --analyze --verbose --table='foo(bar)' xyzzy</userinput> +</screen></para> + + <para> + To repack all tables in the <literal>foo</literal> and <literal>bar</literal> schemas + in a database named <literal>xyzzy</literal>: +<screen> +<prompt>$ </prompt><userinput>pg_repackdb --schema='foo' --schema='bar' xyzzy</userinput> +</screen></para> + + + </refsect1> + + <refsect1> + <title>See Also</title> + + <simplelist type="inline"> + <member><xref linkend="sql-repack"/></member> + </simplelist> + </refsect1> + +</refentry> diff --git a/doc/src/sgml/ref/repack.sgml b/doc/src/sgml/ref/repack.sgml new file mode 100644 index 00000000000..0e1116eae85 --- /dev/null +++ b/doc/src/sgml/ref/repack.sgml @@ -0,0 +1,319 @@ +<!-- +doc/src/sgml/ref/repack.sgml +PostgreSQL documentation +--> + +<refentry id="sql-repack"> + <indexterm zone="sql-repack"> + <primary>REPACK</primary> + </indexterm> + + <refmeta> + <refentrytitle>REPACK</refentrytitle> + <manvolnum>7</manvolnum> + <refmiscinfo>SQL - Language Statements</refmiscinfo> + </refmeta> + + <refnamediv> + <refname>REPACK</refname> + <refpurpose>rewrite a table to reclaim disk space</refpurpose> + </refnamediv> + + <refsynopsisdiv> +<synopsis> +REPACK [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <replaceable class="parameter">table_and_columns</replaceable> [ USING INDEX [ <replaceable class="parameter">index_name</replaceable> ] ] ] + +<phrase>where <replaceable class="parameter">option</replaceable> can be one of:</phrase> + + VERBOSE [ <replaceable class="parameter">boolean</replaceable> ] + ANALYZE [ <replaceable class="parameter">boolean</replaceable> ] + +<phrase>and <replaceable class="parameter">table_and_columns</replaceable> is:</phrase> + + <replaceable class="parameter">table_name</replaceable> [ ( <replaceable class="parameter">column_name</replaceable> [, ...] ) ] +</synopsis> + </refsynopsisdiv> + + <refsect1> + <title>Description</title> + + <para> + <command>REPACK</command> reclaims storage occupied by dead + tuples. Unlike <command>VACUUM</command>, it does so by rewriting the + entire contents of the table specified + by <replaceable class="parameter">table_name</replaceable> into a new disk + file with no extra space (except for the space guaranteed by + the <literal>fillfactor</literal> storage parameter), allowing unused space + to be returned to the operating system. + </para> + + <para> + Without + a <replaceable class="parameter">table_name</replaceable>, <command>REPACK</command> + processes every table and materialized view in the current database that + the current user has the <literal>MAINTAIN</literal> privilege on. This + form of <command>REPACK</command> cannot be executed inside a transaction + block. + </para> + + <para> + If a <literal>USING INDEX</literal> clause is specified, the rows are + physically reordered based on information from an index. Please see the + notes on clustering below. + </para> + + <para> + When a table is being repacked, an <literal>ACCESS EXCLUSIVE</literal> lock + is acquired on it. This prevents any other database operations (both reads + and writes) from operating on the table until the <command>REPACK</command> + is finished. + </para> + + <refsect2 id="sql-repack-notes-on-clustering" xreflabel="Notes on Clustering"> + <title>Notes on Clustering</title> + + <para> + If the <literal>USING INDEX</literal> clause is specified, the rows in + the table are physically reordered following an index: if an index name + is specified in the command, then that index is used; if no index name + is specified, then the index that has been configured as the index to + cluster on. If no index has been configured in this way, an error is + thrown. The index given in the <literal>USING INDEX</literal> clause + is configured as the index to cluster on, as well as an index given + to the <command>CLUSTER</command> command. An index can be set + manually using <command>ALTER TABLE ... CLUSTER ON</command>, and reset + with <command>ALTER TABLE ... SET WITHOUT CLUSTER</command>. + </para> + + <para> + If no table name is specified in <command>REPACK USING INDEX</command>, + all tables which have a clustering index defined and which the calling + user has privileges for are processed. + </para> + + <para> + Clustering is a one-time operation: when the table is + subsequently updated, the changes are not clustered. That is, no attempt + is made to store new or updated rows according to their index order. (If + one wishes, one can periodically recluster by issuing the command again. + Also, setting the table's <literal>fillfactor</literal> storage parameter + to less than 100% can aid in preserving cluster ordering during updates, + since updated rows are kept on the same page if enough space is available + there.) + </para> + + <para> + In cases where you are accessing single rows randomly within a table, the + actual order of the data in the table is unimportant. However, if you tend + to access some data more than others, and there is an index that groups + them together, you will benefit from using clustering. If + you are requesting a range of indexed values from a table, or a single + indexed value that has multiple rows that match, + <command>REPACK</command> will help because once the index identifies the + table page for the first row that matches, all other rows that match are + probably already on the same table page, and so you save disk accesses and + speed up the query. + </para> + + <para> + <command>REPACK</command> can re-sort the table using either an index scan + on the specified index (if the index is a b-tree), or a sequential scan + followed by sorting. It will attempt to choose the method that will be + faster, based on planner cost parameters and available statistical + information. + </para> + + <para> + Because the planner records statistics about the ordering of tables, it is + advisable to + run <link linkend="sql-analyze"><command>ANALYZE</command></link> on the + newly repacked table. Otherwise, the planner might make poor choices of + query plans. + </para> + </refsect2> + + <refsect2 id="sql-repack-notes-on-resources" xreflabel="Notes on Resources"> + <title>Notes on Resources</title> + + <para> + When an index scan or a sequential scan without sort is used, a temporary + copy of the table is created that contains the table data in the index + order. Temporary copies of each index on the table are created as well. + Therefore, you need free space on disk at least equal to the sum of the + table size and the index sizes. + </para> + + <para> + When a sequential scan and sort is used, a temporary sort file is also + created, so that the peak temporary space requirement is as much as double + the table size, plus the index sizes. This method is often faster than + the index scan method, but if the disk space requirement is intolerable, + you can disable this choice by temporarily setting + <xref linkend="guc-enable-sort"/> to <literal>off</literal>. + </para> + + <para> + It is advisable to set <xref linkend="guc-maintenance-work-mem"/> to a + reasonably large value (but not more than the amount of RAM you can + dedicate to the <command>REPACK</command> operation) before repacking. + </para> + </refsect2> + + </refsect1> + + <refsect1> + <title>Parameters</title> + + <variablelist> + <varlistentry> + <term><replaceable class="parameter">table_name</replaceable></term> + <listitem> + <para> + The name (possibly schema-qualified) of a table. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><replaceable class="parameter">column_name</replaceable></term> + <listitem> + <para> + The name of a specific column to analyze. Defaults to all columns. + If a column list is specific, <literal>ANALYZE</literal> must also + be specified. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><replaceable class="parameter">index_name</replaceable></term> + <listitem> + <para> + The name of an index. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><literal>VERBOSE</literal></term> + <listitem> + <para> + Prints a progress report as each table is repacked + at <literal>INFO</literal> level. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><literal>ANALYZE</literal></term> + <term><literal>ANALYSE</literal></term> + <listitem> + <para> + Applies <xref linkend="sql-analyze"/> on the table after repacking. This is + currently only supported when a single (non-partitioned) table is specified. + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><replaceable class="parameter">boolean</replaceable></term> + <listitem> + <para> + Specifies whether the selected option should be turned on or off. + You can write <literal>TRUE</literal>, <literal>ON</literal>, or + <literal>1</literal> to enable the option, and <literal>FALSE</literal>, + <literal>OFF</literal>, or <literal>0</literal> to disable it. The + <replaceable class="parameter">boolean</replaceable> value can also + be omitted, in which case <literal>TRUE</literal> is assumed. + </para> + </listitem> + </varlistentry> + </variablelist> + </refsect1> + + <refsect1> + <title>Notes</title> + + <para> + To repack a table, one must have the <literal>MAINTAIN</literal> privilege + on the table. + </para> + + <para> + While <command>REPACK</command> is running, the <xref + linkend="guc-search-path"/> is temporarily changed to <literal>pg_catalog, + pg_temp</literal>. + </para> + + <para> + Each backend running <command>REPACK</command> will report its progress + in the <structname>pg_stat_progress_repack</structname> view. See + <xref linkend="repack-progress-reporting"/> for details. + </para> + + <para> + Repacking a partitioned table repacks each of its partitions. If an index + is specified, each partition is repacked using the partition of that + index. <command>REPACK</command> on a partitioned table cannot be executed + inside a transaction block. + </para> + + </refsect1> + + <refsect1> + <title>Examples</title> + + <para> + Repack the table <literal>employees</literal>: +<programlisting> +REPACK employees; +</programlisting> + </para> + + <para> + Repack the table <literal>employees</literal> on the basis of its + index <literal>employees_ind</literal> (Since index is used here, this is + effectively clustering): +<programlisting> +REPACK employees USING INDEX employees_ind; +</programlisting> + </para> + + <para> + Repack the table <literal>cases</literal> on physical ordering, + running an <command>ANALYZE</command> on the given columns once + repacking is done, showing informational messages: +<programlisting> +REPACK (ANALYZE, VERBOSE) cases (district, case_nr); +</programlisting> + </para> + + <para> + Repack all tables in the database on which you have + the <literal>MAINTAIN</literal> privilege: +<programlisting> +REPACK; +</programlisting> + </para> + + <para> + Repack all tables for which a clustering index has previously been + configured on which you have the <literal>MAINTAIN</literal> privilege, + showing informational messages: +<programlisting> +REPACK (VERBOSE) USING INDEX; +</programlisting> + </para> + + </refsect1> + + <refsect1> + <title>Compatibility</title> + + <para> + There is no <command>REPACK</command> statement in the SQL standard. + </para> + + </refsect1> + +</refentry> diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index bd5dcaf86a5..062b658cfcd 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -25,7 +25,6 @@ VACUUM [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <re <phrase>where <replaceable class="parameter">option</replaceable> can be one of:</phrase> - FULL [ <replaceable class="parameter">boolean</replaceable> ] FREEZE [ <replaceable class="parameter">boolean</replaceable> ] VERBOSE [ <replaceable class="parameter">boolean</replaceable> ] ANALYZE [ <replaceable class="parameter">boolean</replaceable> ] @@ -39,6 +38,7 @@ VACUUM [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <re SKIP_DATABASE_STATS [ <replaceable class="parameter">boolean</replaceable> ] ONLY_DATABASE_STATS [ <replaceable class="parameter">boolean</replaceable> ] BUFFER_USAGE_LIMIT <replaceable class="parameter">size</replaceable> + FULL [ <replaceable class="parameter">boolean</replaceable> ] <phrase>and <replaceable class="parameter">table_and_columns</replaceable> is:</phrase> @@ -95,20 +95,6 @@ VACUUM [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <re <title>Parameters</title> <variablelist> - <varlistentry> - <term><literal>FULL</literal></term> - <listitem> - <para> - Selects <quote>full</quote> vacuum, which can reclaim more - space, but takes much longer and exclusively locks the table. - This method also requires extra disk space, since it writes a - new copy of the table and doesn't release the old copy until - the operation is complete. Usually this should only be used when a - significant amount of space needs to be reclaimed from within the table. - </para> - </listitem> - </varlistentry> - <varlistentry> <term><literal>FREEZE</literal></term> <listitem> @@ -362,6 +348,23 @@ VACUUM [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ] [ <re </listitem> </varlistentry> + <varlistentry> + <term><literal>FULL</literal></term> + <listitem> + <para> + This option, which is deprecated, makes <command>VACUUM</command> + behave like <command>REPACK</command> without a + <literal>USING INDEX</literal> clause. + This method of compacting the table takes much longer than + <command>VACUUM</command> and exclusively locks the table. + This method also requires extra disk space, since it writes a + new copy of the table and doesn't release the old copy until + the operation is complete. Usually this should only be used when a + significant amount of space needs to be reclaimed from within the table. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><replaceable class="parameter">boolean</replaceable></term> <listitem> diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index ff85ace83fc..2ee08e21f41 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -195,6 +195,7 @@ &refreshMaterializedView; &reindex; &releaseSavepoint; + &repack; &reset; &revoke; &rollback; @@ -257,6 +258,7 @@ &pgIsready; &pgReceivewal; &pgRecvlogical; + &pgRepackdb; &pgRestore; &pgVerifyBackup; &psqlRef; diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index bcbac844bb6..79f9de5d760 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -741,13 +741,13 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, if (OldIndex != NULL && !use_sort) { const int ci_index[] = { - PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_INDEX_RELID + PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_INDEX_RELID }; int64 ci_val[2]; /* Set phase and OIDOldIndex to columns */ - ci_val[0] = PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP; + ci_val[0] = PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP; ci_val[1] = RelationGetRelid(OldIndex); pgstat_progress_update_multi_param(2, ci_index, ci_val); @@ -759,15 +759,15 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, else { /* In scan-and-sort mode and also VACUUM FULL, set phase */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP); tableScan = table_beginscan(OldHeap, SnapshotAny, 0, (ScanKey) NULL); heapScan = (HeapScanDesc) tableScan; indexScan = NULL; /* Set total heap blocks */ - pgstat_progress_update_param(PROGRESS_CLUSTER_TOTAL_HEAP_BLKS, + pgstat_progress_update_param(PROGRESS_REPACK_TOTAL_HEAP_BLKS, heapScan->rs_nblocks); } @@ -809,7 +809,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, * is manually updated to the correct value when the table * scan finishes. */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_BLKS_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_BLKS_SCANNED, heapScan->rs_nblocks); break; } @@ -825,7 +825,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, */ if (prev_cblock != heapScan->rs_cblock) { - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_BLKS_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_BLKS_SCANNED, (heapScan->rs_cblock + heapScan->rs_nblocks - heapScan->rs_startblock @@ -912,14 +912,14 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, * In scan-and-sort mode, report increase in number of tuples * scanned */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_SCANNED, *num_tuples); } else { const int ct_index[] = { - PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED, - PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN + PROGRESS_REPACK_HEAP_TUPLES_SCANNED, + PROGRESS_REPACK_HEAP_TUPLES_WRITTEN }; int64 ct_val[2]; @@ -952,14 +952,14 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, double n_tuples = 0; /* Report that we are now sorting tuples */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SORT_TUPLES); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SORT_TUPLES); tuplesort_performsort(tuplesort); /* Report that we are now writing new heap */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_WRITE_NEW_HEAP); for (;;) { @@ -977,7 +977,7 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, values, isnull, rwstate); /* Report n_tuples */ - pgstat_progress_update_param(PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN, + pgstat_progress_update_param(PROGRESS_REPACK_HEAP_TUPLES_WRITTEN, n_tuples); } diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 5d9db167e59..08d4b8e44d7 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -4079,7 +4079,7 @@ reindex_relation(const ReindexStmt *stmt, Oid relid, int flags, Assert(!ReindexIsProcessingIndex(indexOid)); /* Set index rebuild count */ - pgstat_progress_update_param(PROGRESS_CLUSTER_INDEX_REBUILD_COUNT, + pgstat_progress_update_param(PROGRESS_REPACK_INDEX_REBUILD_COUNT, i); i++; } diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index dec8df4f8ee..1ad30116631 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1269,14 +1269,15 @@ CREATE VIEW pg_stat_progress_vacuum AS FROM pg_stat_get_progress_info('VACUUM') AS S LEFT JOIN pg_database D ON S.datid = D.oid; -CREATE VIEW pg_stat_progress_cluster AS +CREATE VIEW pg_stat_progress_repack AS SELECT S.pid AS pid, S.datid AS datid, D.datname AS datname, S.relid AS relid, CASE S.param1 WHEN 1 THEN 'CLUSTER' - WHEN 2 THEN 'VACUUM FULL' + WHEN 2 THEN 'REPACK' + WHEN 3 THEN 'VACUUM FULL' END AS command, CASE S.param2 WHEN 0 THEN 'initializing' WHEN 1 THEN 'seq scanning heap' @@ -1287,15 +1288,35 @@ CREATE VIEW pg_stat_progress_cluster AS WHEN 6 THEN 'rebuilding index' WHEN 7 THEN 'performing final cleanup' END AS phase, - CAST(S.param3 AS oid) AS cluster_index_relid, + CAST(S.param3 AS oid) AS repack_index_relid, S.param4 AS heap_tuples_scanned, S.param5 AS heap_tuples_written, S.param6 AS heap_blks_total, S.param7 AS heap_blks_scanned, S.param8 AS index_rebuild_count - FROM pg_stat_get_progress_info('CLUSTER') AS S + FROM pg_stat_get_progress_info('REPACK') AS S LEFT JOIN pg_database D ON S.datid = D.oid; +-- This view is as the one above, except for renaming a column and avoiding +-- 'REPACK' as a command name to report. +CREATE VIEW pg_stat_progress_cluster AS + SELECT + pid, + datid, + datname, + relid, + CASE WHEN command IN ('CLUSTER', 'VACUUM FULL') THEN command + WHEN repack_index_relid = 0 THEN 'VACUUM FULL' + ELSE 'CLUSTER' END AS command, + phase, + repack_index_relid AS cluster_index_relid, + heap_tuples_scanned, + heap_tuples_written, + heap_blks_total, + heap_blks_scanned, + index_rebuild_count + FROM pg_stat_progress_repack; + CREATE VIEW pg_stat_progress_create_index AS SELECT S.pid AS pid, S.datid AS datid, D.datname AS datname, diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index b55221d44cd..18bee52a4ee 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -67,27 +67,36 @@ typedef struct Oid indexOid; } RelToCluster; - -static void cluster_multiple_rels(List *rtcs, ClusterParams *params); -static void rebuild_relation(Relation OldHeap, Relation index, bool verbose); +static bool cluster_rel_recheck(RepackCommand cmd, Relation OldHeap, + Oid indexOid, Oid userid, int options); +static void rebuild_relation(RepackCommand cmd, + Relation OldHeap, Relation index, bool verbose); static void copy_table_data(Relation NewHeap, Relation OldHeap, Relation OldIndex, bool verbose, bool *pSwapToastByContent, TransactionId *pFreezeXid, MultiXactId *pCutoffMulti); -static List *get_tables_to_cluster(MemoryContext cluster_context); -static List *get_tables_to_cluster_partitioned(MemoryContext cluster_context, - Oid indexOid); -static bool cluster_is_permitted_for_relation(Oid relid, Oid userid); +static List *get_tables_to_repack(RepackCommand cmd, bool usingindex, + MemoryContext permcxt); +static List *get_tables_to_repack_partitioned(RepackCommand cmd, + Oid relid, bool rel_is_index, + MemoryContext permcxt); +static bool cluster_is_permitted_for_relation(RepackCommand cmd, + Oid relid, Oid userid); +static Relation process_single_relation(RepackStmt *stmt, + ClusterParams *params); +static Oid determine_clustered_index(Relation rel, bool usingindex, + const char *indexname); +static const char *RepackCommandAsString(RepackCommand cmd); -/*--------------------------------------------------------------------------- - * This cluster code allows for clustering multiple tables at once. Because +/* + * The repack code allows for processing multiple tables at once. Because * of this, we cannot just run everything on a single transaction, or we * would be forced to acquire exclusive locks on all the tables being * clustered, simultaneously --- very likely leading to deadlock. * - * To solve this we follow a similar strategy to VACUUM code, - * clustering each relation in a separate transaction. For this to work, - * we need to: + * To solve this we follow a similar strategy to VACUUM code, processing each + * relation in a separate transaction. For this to work, we need to: + * * - provide a separate memory context so that we can pass information in * a way that survives across transactions * - start a new transaction every time a new relation is clustered @@ -98,197 +107,165 @@ static bool cluster_is_permitted_for_relation(Oid relid, Oid userid); * * The single-relation case does not have any such overhead. * - * We also allow a relation to be specified without index. In that case, - * the indisclustered bit will be looked up, and an ERROR will be thrown - * if there is no index with the bit set. - *--------------------------------------------------------------------------- + * We also allow a relation to be repacked following an index, but without + * naming a specific one. In that case, the indisclustered bit will be + * looked up, and an ERROR will be thrown if no so-marked index is found. */ void -cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel) +ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel) { - ListCell *lc; ClusterParams params = {0}; - bool verbose = false; Relation rel = NULL; - Oid indexOid = InvalidOid; - MemoryContext cluster_context; + MemoryContext repack_context; List *rtcs; /* Parse option list */ - foreach(lc, stmt->params) + foreach_node(DefElem, opt, stmt->params) { - DefElem *opt = (DefElem *) lfirst(lc); - if (strcmp(opt->defname, "verbose") == 0) - verbose = defGetBoolean(opt); + params.options |= defGetBoolean(opt) ? CLUOPT_VERBOSE : 0; + else if (strcmp(opt->defname, "analyze") == 0 || + strcmp(opt->defname, "analyse") == 0) + params.options |= defGetBoolean(opt) ? CLUOPT_ANALYZE : 0; else ereport(ERROR, - (errcode(ERRCODE_SYNTAX_ERROR), - errmsg("unrecognized CLUSTER option \"%s\"", - opt->defname), - parser_errposition(pstate, opt->location))); + errcode(ERRCODE_SYNTAX_ERROR), + errmsg("unrecognized %s option \"%s\"", + RepackCommandAsString(stmt->command), + opt->defname), + parser_errposition(pstate, opt->location)); } - params.options = (verbose ? CLUOPT_VERBOSE : 0); - + /* + * If a single relation is specified, process it and we're done ... unless + * the relation is a partitioned table, in which case we fall through. + */ if (stmt->relation != NULL) { - /* This is the single-relation case. */ - Oid tableOid; - - /* - * Find, lock, and check permissions on the table. We obtain - * AccessExclusiveLock right away to avoid lock-upgrade hazard in the - * single-transaction case. - */ - tableOid = RangeVarGetRelidExtended(stmt->relation, - AccessExclusiveLock, - 0, - RangeVarCallbackMaintainsTable, - NULL); - rel = table_open(tableOid, NoLock); - - /* - * Reject clustering a remote temp table ... their local buffer - * manager is not going to cope. - */ - if (RELATION_IS_OTHER_TEMP(rel)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster temporary tables of other sessions"))); - - if (stmt->indexname == NULL) - { - ListCell *index; - - /* We need to find the index that has indisclustered set. */ - foreach(index, RelationGetIndexList(rel)) - { - indexOid = lfirst_oid(index); - if (get_index_isclustered(indexOid)) - break; - indexOid = InvalidOid; - } - - if (!OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("there is no previously clustered index for table \"%s\"", - stmt->relation->relname))); - } - else - { - /* - * The index is expected to be in the same namespace as the - * relation. - */ - indexOid = get_relname_relid(stmt->indexname, - rel->rd_rel->relnamespace); - if (!OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("index \"%s\" for table \"%s\" does not exist", - stmt->indexname, stmt->relation->relname))); - } - - /* For non-partitioned tables, do what we came here to do. */ - if (rel->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) - { - cluster_rel(rel, indexOid, ¶ms); - /* cluster_rel closes the relation, but keeps lock */ - - return; - } + rel = process_single_relation(stmt, ¶ms); + if (rel == NULL) + return; /* all done */ } + /* + * Don't allow ANALYZE in the multiple-relation case for now. Maybe we + * can add support for this later. + */ + if (params.options & CLUOPT_ANALYZE) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot %s multiple tables", "REPACK (ANALYZE)")); + /* * By here, we know we are in a multi-table situation. In order to avoid * holding locks for too long, we want to process each table in its own * transaction. This forces us to disallow running inside a user * transaction block. */ - PreventInTransactionBlock(isTopLevel, "CLUSTER"); + PreventInTransactionBlock(isTopLevel, RepackCommandAsString(stmt->command)); /* Also, we need a memory context to hold our list of relations */ - cluster_context = AllocSetContextCreate(PortalContext, - "Cluster", - ALLOCSET_DEFAULT_SIZES); + repack_context = AllocSetContextCreate(PortalContext, + "Repack", + ALLOCSET_DEFAULT_SIZES); + + params.options |= CLUOPT_RECHECK; /* - * Either we're processing a partitioned table, or we were not given any - * table name at all. In either case, obtain a list of relations to - * process. - * - * In the former case, an index name must have been given, so we don't - * need to recheck its "indisclustered" bit, but we have to check that it - * is an index that we can cluster on. In the latter case, we set the - * option bit to have indisclustered verified. - * - * Rechecking the relation itself is necessary here in all cases. + * If we don't have a relation yet, determine a relation list. If we do, + * then it must be a partitioned table, and we want to process its + * partitions. */ - params.options |= CLUOPT_RECHECK; - if (rel != NULL) + if (rel == NULL) { - Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); - check_index_is_clusterable(rel, indexOid, AccessShareLock); - rtcs = get_tables_to_cluster_partitioned(cluster_context, indexOid); - - /* close relation, releasing lock on parent table */ - table_close(rel, AccessExclusiveLock); + Assert(stmt->indexname == NULL); + rtcs = get_tables_to_repack(stmt->command, stmt->usingindex, + repack_context); } else { - rtcs = get_tables_to_cluster(cluster_context); - params.options |= CLUOPT_RECHECK_ISCLUSTERED; + Oid relid; + bool rel_is_index; + + Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); + + /* + * If USING INDEX was specified, resolve the index name now and pass + * it down. + */ + if (stmt->usingindex) + { + /* + * If no index name was specified when repacking a partitioned + * table, punt for now. Maybe we can improve this later. + */ + if (!stmt->indexname) + ereport(ERROR, + errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("there is no previously clustered index for table \"%s\"", + RelationGetRelationName(rel))); + + relid = determine_clustered_index(rel, stmt->usingindex, + stmt->indexname); + if (!OidIsValid(relid)) + elog(ERROR, "unable to determine index to cluster on"); + /* XXX is this the right place for this check? */ + check_index_is_clusterable(rel, relid, AccessExclusiveLock); + rel_is_index = true; + } + else + { + relid = RelationGetRelid(rel); + rel_is_index = false; + } + + rtcs = get_tables_to_repack_partitioned(stmt->command, + relid, rel_is_index, + repack_context); + + /* close parent relation, releasing lock on it */ + table_close(rel, AccessExclusiveLock); + rel = NULL; } - /* Do the job. */ - cluster_multiple_rels(rtcs, ¶ms); - - /* Start a new transaction for the cleanup work. */ - StartTransactionCommand(); - - /* Clean up working storage */ - MemoryContextDelete(cluster_context); -} - -/* - * Given a list of relations to cluster, process each of them in a separate - * transaction. - * - * We expect to be in a transaction at start, but there isn't one when we - * return. - */ -static void -cluster_multiple_rels(List *rtcs, ClusterParams *params) -{ - ListCell *lc; - /* Commit to get out of starting transaction */ PopActiveSnapshot(); CommitTransactionCommand(); /* Cluster the tables, each in a separate transaction */ - foreach(lc, rtcs) + Assert(rel == NULL); + foreach_ptr(RelToCluster, rtc, rtcs) { - RelToCluster *rtc = (RelToCluster *) lfirst(lc); - Relation rel; - /* Start a new transaction for each relation. */ StartTransactionCommand(); + /* + * Open the target table, coping with the case where it has been + * dropped. + */ + rel = try_table_open(rtc->tableOid, AccessExclusiveLock); + if (rel == NULL) + { + CommitTransactionCommand(); + continue; + } + /* functions in indexes may want a snapshot set */ PushActiveSnapshot(GetTransactionSnapshot()); - rel = table_open(rtc->tableOid, AccessExclusiveLock); - /* Process this table */ - cluster_rel(rel, rtc->indexOid, params); + cluster_rel(stmt->command, rel, rtc->indexOid, ¶ms); /* cluster_rel closes the relation, but keeps lock */ PopActiveSnapshot(); CommitTransactionCommand(); } + + /* Start a new transaction for the cleanup work. */ + StartTransactionCommand(); + + /* Clean up working storage */ + MemoryContextDelete(repack_context); } /* @@ -304,11 +281,14 @@ cluster_multiple_rels(List *rtcs, ClusterParams *params) * them incrementally while we load the table. * * If indexOid is InvalidOid, the table will be rewritten in physical order - * instead of index order. This is the new implementation of VACUUM FULL, - * and error messages should refer to the operation as VACUUM not CLUSTER. + * instead of index order. + * + * 'cmd' indicates which command is being executed, to be used for error + * messages. */ void -cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) +cluster_rel(RepackCommand cmd, Relation OldHeap, Oid indexOid, + ClusterParams *params) { Oid tableOid = RelationGetRelid(OldHeap); Oid save_userid; @@ -323,13 +303,8 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) /* Check for user-requested abort. */ CHECK_FOR_INTERRUPTS(); - pgstat_progress_start_command(PROGRESS_COMMAND_CLUSTER, tableOid); - if (OidIsValid(indexOid)) - pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, - PROGRESS_CLUSTER_COMMAND_CLUSTER); - else - pgstat_progress_update_param(PROGRESS_CLUSTER_COMMAND, - PROGRESS_CLUSTER_COMMAND_VACUUM_FULL); + pgstat_progress_start_command(PROGRESS_COMMAND_REPACK, tableOid); + pgstat_progress_update_param(PROGRESS_REPACK_COMMAND, cmd); /* * Switch to the table owner's userid, so that any index functions are run @@ -350,86 +325,38 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) * *must* skip the one on indisclustered since it would reject an attempt * to cluster a not-previously-clustered index. */ - if (recheck) - { - /* Check that the user still has privileges for the relation */ - if (!cluster_is_permitted_for_relation(tableOid, save_userid)) - { - relation_close(OldHeap, AccessExclusiveLock); - goto out; - } - - /* - * Silently skip a temp table for a remote session. Only doing this - * check in the "recheck" case is appropriate (which currently means - * somebody is executing a database-wide CLUSTER or on a partitioned - * table), because there is another check in cluster() which will stop - * any attempt to cluster remote temp tables by name. There is - * another check in cluster_rel which is redundant, but we leave it - * for extra safety. - */ - if (RELATION_IS_OTHER_TEMP(OldHeap)) - { - relation_close(OldHeap, AccessExclusiveLock); - goto out; - } - - if (OidIsValid(indexOid)) - { - /* - * Check that the index still exists - */ - if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid))) - { - relation_close(OldHeap, AccessExclusiveLock); - goto out; - } - - /* - * Check that the index is still the one with indisclustered set, - * if needed. - */ - if ((params->options & CLUOPT_RECHECK_ISCLUSTERED) != 0 && - !get_index_isclustered(indexOid)) - { - relation_close(OldHeap, AccessExclusiveLock); - goto out; - } - } - } + if (recheck && + !cluster_rel_recheck(cmd, OldHeap, indexOid, save_userid, + params->options)) + goto out; /* - * We allow VACUUM FULL, but not CLUSTER, on shared catalogs. CLUSTER - * would work in most respects, but the index would only get marked as - * indisclustered in the current database, leading to unexpected behavior - * if CLUSTER were later invoked in another database. + * We allow repacking shared catalogs only when not using an index. It + * would work to use an index in most respects, but the index would only + * get marked as indisclustered in the current database, leading to + * unexpected behavior if CLUSTER were later invoked in another database. */ if (OidIsValid(indexOid) && OldHeap->rd_rel->relisshared) ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster a shared catalog"))); + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot run %s on a shared catalog", + RepackCommandAsString(cmd))); /* * Don't process temp tables of other backends ... their local buffer * manager is not going to cope. */ if (RELATION_IS_OTHER_TEMP(OldHeap)) - { - if (OidIsValid(indexOid)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot cluster temporary tables of other sessions"))); - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot vacuum temporary tables of other sessions"))); - } + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot run %s on temporary tables of other sessions", + RepackCommandAsString(cmd))); /* * Also check for active uses of the relation in the current transaction, * including open scans and pending AFTER trigger events. */ - CheckTableNotInUse(OldHeap, OidIsValid(indexOid) ? "CLUSTER" : "VACUUM"); + CheckTableNotInUse(OldHeap, RepackCommandAsString(cmd)); /* Check heap and index are valid to cluster on */ if (OidIsValid(indexOid)) @@ -442,6 +369,24 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) else index = NULL; + /* + * When allow_system_table_mods is turned off, we disallow repacking a + * catalog on a particular index unless that's already the clustered index + * for that catalog. + * + * XXX We don't check for this in CLUSTER, because it's historically been + * allowed. + */ + if (cmd != REPACK_COMMAND_CLUSTER && + !allowSystemTableMods && OidIsValid(indexOid) && + IsCatalogRelation(OldHeap) && !index->rd_index->indisclustered) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied: \"%s\" is a system catalog", + RelationGetRelationName(OldHeap)), + errdetail("System catalogs can only be clustered by the index they're already clustered on, if any, unless \"%s\" is enabled.", + "allow_system_table_mods")); + /* * Quietly ignore the request if this is a materialized view which has not * been populated from its query. No harm is done because there is no data @@ -469,7 +414,7 @@ cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params) TransferPredicateLocksToHeapRelation(OldHeap); /* rebuild_relation does all the dirty work */ - rebuild_relation(OldHeap, index, verbose); + rebuild_relation(cmd, OldHeap, index, verbose); /* rebuild_relation closes OldHeap, and index if valid */ out: @@ -482,6 +427,63 @@ out: pgstat_progress_end_command(); } +/* + * Check if the table (and its index) still meets the requirements of + * cluster_rel(). + */ +static bool +cluster_rel_recheck(RepackCommand cmd, Relation OldHeap, Oid indexOid, + Oid userid, int options) +{ + Oid tableOid = RelationGetRelid(OldHeap); + + /* Check that the user still has privileges for the relation */ + if (!cluster_is_permitted_for_relation(cmd, tableOid, userid)) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + + /* + * Silently skip a temp table for a remote session. Only doing this check + * in the "recheck" case is appropriate (which currently means somebody is + * executing a database-wide CLUSTER or on a partitioned table), because + * there is another check in cluster() which will stop any attempt to + * cluster remote temp tables by name. There is another check in + * cluster_rel which is redundant, but we leave it for extra safety. + */ + if (RELATION_IS_OTHER_TEMP(OldHeap)) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + + if (OidIsValid(indexOid)) + { + /* + * Check that the index still exists + */ + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid))) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + + /* + * Check that the index is still the one with indisclustered set, if + * needed. + */ + if ((options & CLUOPT_RECHECK_ISCLUSTERED) != 0 && + !get_index_isclustered(indexOid)) + { + relation_close(OldHeap, AccessExclusiveLock); + return false; + } + } + + return true; +} + /* * Verify that the specified heap and index are valid to cluster on * @@ -626,7 +628,8 @@ mark_index_clustered(Relation rel, Oid indexOid, bool is_internal) * On exit, they are closed, but locks on them are not released. */ static void -rebuild_relation(Relation OldHeap, Relation index, bool verbose) +rebuild_relation(RepackCommand cmd, + Relation OldHeap, Relation index, bool verbose) { Oid tableOid = RelationGetRelid(OldHeap); Oid accessMethod = OldHeap->rd_rel->relam; @@ -642,8 +645,8 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose) Assert(CheckRelationLockedByMe(OldHeap, AccessExclusiveLock, false) && (index == NULL || CheckRelationLockedByMe(index, AccessExclusiveLock, false))); - if (index) - /* Mark the correct index as clustered */ + /* for CLUSTER or REPACK USING INDEX, mark the index as the one to use */ + if (index != NULL) mark_index_clustered(OldHeap, RelationGetRelid(index), true); /* Remember info about rel before closing OldHeap */ @@ -958,20 +961,20 @@ copy_table_data(Relation NewHeap, Relation OldHeap, Relation OldIndex, bool verb /* Log what we're doing */ if (OldIndex != NULL && !use_sort) ereport(elevel, - (errmsg("clustering \"%s.%s\" using index scan on \"%s\"", - nspname, - RelationGetRelationName(OldHeap), - RelationGetRelationName(OldIndex)))); + errmsg("repacking \"%s.%s\" using index scan on \"%s\"", + nspname, + RelationGetRelationName(OldHeap), + RelationGetRelationName(OldIndex))); else if (use_sort) ereport(elevel, - (errmsg("clustering \"%s.%s\" using sequential scan and sort", - nspname, - RelationGetRelationName(OldHeap)))); + errmsg("repacking \"%s.%s\" using sequential scan and sort", + nspname, + RelationGetRelationName(OldHeap))); else ereport(elevel, - (errmsg("vacuuming \"%s.%s\"", - nspname, - RelationGetRelationName(OldHeap)))); + errmsg("repacking \"%s.%s\" in physical order", + nspname, + RelationGetRelationName(OldHeap))); /* * Hand off the actual copying to AM specific function, the generic code @@ -1458,8 +1461,8 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, int i; /* Report that we are now swapping relation files */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_SWAP_REL_FILES); /* Zero out possible results from swapped_relation_files */ memset(mapped_tables, 0, sizeof(mapped_tables)); @@ -1509,14 +1512,14 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; /* Report that we are now reindexing relations */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_REBUILD_INDEX); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_REBUILD_INDEX); reindex_relation(NULL, OIDOldHeap, reindex_flags, &reindex_params); /* Report that we are now doing clean up */ - pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, - PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP); + pgstat_progress_update_param(PROGRESS_REPACK_PHASE, + PROGRESS_REPACK_PHASE_FINAL_CLEANUP); /* * If the relation being rebuilt is pg_class, swap_relation_files() @@ -1632,106 +1635,191 @@ finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, } } - /* - * Get a list of tables that the current user has privileges on and - * have indisclustered set. Return the list in a List * of RelToCluster - * (stored in the specified memory context), each one giving the tableOid - * and the indexOid on which the table is already clustered. + * Determine which relations to process, when REPACK/CLUSTER is called + * without specifying a table name. The exact process depends on whether + * USING INDEX was given or not, and in any case we only return tables and + * materialized views that the current user has privileges to repack/cluster. + * + * If USING INDEX was given, we scan pg_index to find those that have + * indisclustered set; if it was not given, scan pg_class and return all + * tables. + * + * Return it as a list of RelToCluster in the given memory context. */ static List * -get_tables_to_cluster(MemoryContext cluster_context) +get_tables_to_repack(RepackCommand cmd, bool usingindex, MemoryContext permcxt) { - Relation indRelation; + Relation catalog; TableScanDesc scan; - ScanKeyData entry; - HeapTuple indexTuple; - Form_pg_index index; - MemoryContext old_context; + HeapTuple tuple; List *rtcs = NIL; - /* - * Get all indexes that have indisclustered set and that the current user - * has the appropriate privileges for. - */ - indRelation = table_open(IndexRelationId, AccessShareLock); - ScanKeyInit(&entry, - Anum_pg_index_indisclustered, - BTEqualStrategyNumber, F_BOOLEQ, - BoolGetDatum(true)); - scan = table_beginscan_catalog(indRelation, 1, &entry); - while ((indexTuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + if (usingindex) { - RelToCluster *rtc; + ScanKeyData entry; - index = (Form_pg_index) GETSTRUCT(indexTuple); + catalog = table_open(IndexRelationId, AccessShareLock); + ScanKeyInit(&entry, + Anum_pg_index_indisclustered, + BTEqualStrategyNumber, F_BOOLEQ, + BoolGetDatum(true)); + scan = table_beginscan_catalog(catalog, 1, &entry); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + RelToCluster *rtc; + Form_pg_index index; + MemoryContext oldcxt; - if (!cluster_is_permitted_for_relation(index->indrelid, GetUserId())) - continue; + index = (Form_pg_index) GETSTRUCT(tuple); - /* Use a permanent memory context for the result list */ - old_context = MemoryContextSwitchTo(cluster_context); + /* + * Try to obtain a light lock on the index's table, to ensure it + * doesn't go away while we collect the list. If we cannot, just + * disregard it. + */ + if (!ConditionalLockRelationOid(index->indrelid, AccessShareLock)) + continue; - rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); - rtc->tableOid = index->indrelid; - rtc->indexOid = index->indexrelid; - rtcs = lappend(rtcs, rtc); + /* Verify that the table still exists */ + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(index->indrelid))) + { + /* Release useless lock */ + UnlockRelationOid(index->indrelid, AccessShareLock); + continue; + } - MemoryContextSwitchTo(old_context); + if (!cluster_is_permitted_for_relation(cmd, index->indrelid, + GetUserId())) + continue; + + /* Use a permanent memory context for the result list */ + oldcxt = MemoryContextSwitchTo(permcxt); + rtc = palloc(sizeof(RelToCluster)); + rtc->tableOid = index->indrelid; + rtc->indexOid = index->indexrelid; + rtcs = lappend(rtcs, rtc); + MemoryContextSwitchTo(oldcxt); + } } - table_endscan(scan); + else + { + catalog = table_open(RelationRelationId, AccessShareLock); + scan = table_beginscan_catalog(catalog, 0, NULL); - relation_close(indRelation, AccessShareLock); + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) + { + RelToCluster *rtc; + Form_pg_class class; + MemoryContext oldcxt; + + class = (Form_pg_class) GETSTRUCT(tuple); + + /* + * Try to obtain a light lock on the table, to ensure it doesn't + * go away while we collect the list. If we cannot, just + * disregard the table. + */ + if (!ConditionalLockRelationOid(class->oid, AccessShareLock)) + continue; + + /* Verify that the table still exists */ + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(class->oid))) + { + /* Release useless lock */ + UnlockRelationOid(class->oid, AccessShareLock); + continue; + } + + /* Can only process plain tables and matviews */ + if (class->relkind != RELKIND_RELATION && + class->relkind != RELKIND_MATVIEW) + continue; + + /* noisily skip rels which the user can't process */ + if (!cluster_is_permitted_for_relation(cmd, class->oid, + GetUserId())) + continue; + + /* Use a permanent memory context for the result list */ + oldcxt = MemoryContextSwitchTo(permcxt); + rtc = palloc(sizeof(RelToCluster)); + rtc->tableOid = class->oid; + rtc->indexOid = InvalidOid; + rtcs = lappend(rtcs, rtc); + MemoryContextSwitchTo(oldcxt); + } + } + + table_endscan(scan); + relation_close(catalog, AccessShareLock); return rtcs; } /* - * Given an index on a partitioned table, return a list of RelToCluster for + * Given a partitioned table or its index, return a list of RelToCluster for * all the children leaves tables/indexes. * * Like expand_vacuum_rel, but here caller must hold AccessExclusiveLock * on the table containing the index. + * + * 'rel_is_index' tells whether 'relid' is that of an index (true) or of the + * owning relation. */ static List * -get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) +get_tables_to_repack_partitioned(RepackCommand cmd, Oid relid, + bool rel_is_index, MemoryContext permcxt) { List *inhoids; - ListCell *lc; List *rtcs = NIL; - MemoryContext old_context; - /* Do not lock the children until they're processed */ - inhoids = find_all_inheritors(indexOid, NoLock, NULL); - - foreach(lc, inhoids) + /* + * Do not lock the children until they're processed. Note that we do hold + * a lock on the parent partitioned table. + */ + inhoids = find_all_inheritors(relid, NoLock, NULL); + foreach_oid(child_oid, inhoids) { - Oid indexrelid = lfirst_oid(lc); - Oid relid = IndexGetRelation(indexrelid, false); + Oid table_oid, + index_oid; RelToCluster *rtc; + MemoryContext oldcxt; - /* consider only leaf indexes */ - if (get_rel_relkind(indexrelid) != RELKIND_INDEX) - continue; + if (rel_is_index) + { + /* consider only leaf indexes */ + if (get_rel_relkind(child_oid) != RELKIND_INDEX) + continue; + + table_oid = IndexGetRelation(child_oid, false); + index_oid = child_oid; + } + else + { + /* consider only leaf relations */ + if (get_rel_relkind(child_oid) != RELKIND_RELATION) + continue; + + table_oid = child_oid; + index_oid = InvalidOid; + } /* * It's possible that the user does not have privileges to CLUSTER the - * leaf partition despite having such privileges on the partitioned - * table. We skip any partitions which the user is not permitted to - * CLUSTER. + * leaf partition despite having them on the partitioned table. Skip + * if so. */ - if (!cluster_is_permitted_for_relation(relid, GetUserId())) + if (!cluster_is_permitted_for_relation(cmd, table_oid, GetUserId())) continue; /* Use a permanent memory context for the result list */ - old_context = MemoryContextSwitchTo(cluster_context); - - rtc = (RelToCluster *) palloc(sizeof(RelToCluster)); - rtc->tableOid = relid; - rtc->indexOid = indexrelid; + oldcxt = MemoryContextSwitchTo(permcxt); + rtc = palloc(sizeof(RelToCluster)); + rtc->tableOid = table_oid; + rtc->indexOid = index_oid; rtcs = lappend(rtcs, rtc); - - MemoryContextSwitchTo(old_context); + MemoryContextSwitchTo(oldcxt); } return rtcs; @@ -1742,13 +1830,167 @@ get_tables_to_cluster_partitioned(MemoryContext cluster_context, Oid indexOid) * function emits a WARNING. */ static bool -cluster_is_permitted_for_relation(Oid relid, Oid userid) +cluster_is_permitted_for_relation(RepackCommand cmd, Oid relid, Oid userid) { + Assert(cmd == REPACK_COMMAND_CLUSTER || cmd == REPACK_COMMAND_REPACK); + if (pg_class_aclcheck(relid, userid, ACL_MAINTAIN) == ACLCHECK_OK) return true; ereport(WARNING, - (errmsg("permission denied to cluster \"%s\", skipping it", - get_rel_name(relid)))); + errmsg("permission denied to execute %s on \"%s\", skipping it", + RepackCommandAsString(cmd), + get_rel_name(relid))); + return false; } + + +/* + * Given a RepackStmt with an indicated relation name, resolve the relation + * name, obtain lock on it, then determine what to do based on the relation + * type: if it's table and not partitioned, repack it as indicated (using an + * existing clustered index, or following the given one), and return NULL. + * + * On the other hand, if the table is partitioned, do nothing further and + * instead return the opened and locked relcache entry, so that caller can + * process the partitions using the multiple-table handling code. In this + * case, if an index name is given, it's up to the caller to resolve it. + */ +static Relation +process_single_relation(RepackStmt *stmt, ClusterParams *params) +{ + Relation rel; + Oid tableOid; + + Assert(stmt->relation != NULL); + Assert(stmt->command == REPACK_COMMAND_CLUSTER || + stmt->command == REPACK_COMMAND_REPACK); + + /* + * Find, lock, and check permissions on the table. We obtain + * AccessExclusiveLock right away to avoid lock-upgrade hazard in the + * single-transaction case. + */ + tableOid = RangeVarGetRelidExtended(stmt->relation->relation, + AccessExclusiveLock, + 0, + RangeVarCallbackMaintainsTable, + NULL); + rel = table_open(tableOid, NoLock); + + /* + * Reject clustering a remote temp table ... their local buffer manager is + * not going to cope. + */ + if (RELATION_IS_OTHER_TEMP(rel)) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot execute %s on temporary tables of other sessions", + RepackCommandAsString(stmt->command))); + + /* + * Make sure ANALYZE is specified if a column list is present. + */ + if ((params->options & CLUOPT_ANALYZE) == 0 && stmt->relation->va_cols != NIL) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ANALYZE option must be specified when a column list is provided")); + + /* + * For partitioned tables, let caller handle this. Otherwise, process it + * here and we're done. + */ + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + return rel; + else + { + Oid indexOid; + + indexOid = determine_clustered_index(rel, stmt->usingindex, + stmt->indexname); + if (OidIsValid(indexOid)) + check_index_is_clusterable(rel, indexOid, AccessExclusiveLock); + cluster_rel(stmt->command, rel, indexOid, params); + + /* Do an analyze, if requested */ + if (params->options & CLUOPT_ANALYZE) + { + VacuumParams vac_params = {0}; + + vac_params.options |= VACOPT_ANALYZE; + if (params->options & CLUOPT_VERBOSE) + vac_params.options |= VACOPT_VERBOSE; + analyze_rel(RelationGetRelid(rel), NULL, vac_params, + stmt->relation->va_cols, true, NULL); + } + + return NULL; + } +} + +/* + * Given a relation and the usingindex/indexname options in a + * REPACK USING INDEX or CLUSTER command, return the OID of the + * index to use for clustering the table. + * + * Caller must hold lock on the relation so that the set of indexes + * doesn't change, and must call check_index_is_clusterable. + */ +static Oid +determine_clustered_index(Relation rel, bool usingindex, const char *indexname) +{ + Oid indexOid; + + if (indexname == NULL && usingindex) + { + /* + * If USING INDEX with no name is given, find a clustered index, or + * error out if none. + */ + indexOid = InvalidOid; + foreach_oid(idxoid, RelationGetIndexList(rel)) + { + if (get_index_isclustered(idxoid)) + { + indexOid = idxoid; + break; + } + } + + if (!OidIsValid(indexOid)) + ereport(ERROR, + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("there is no previously clustered index for table \"%s\"", + RelationGetRelationName(rel))); + } + else if (indexname != NULL) + { + /* An index was specified; obtain its OID. */ + indexOid = get_relname_relid(indexname, rel->rd_rel->relnamespace); + if (!OidIsValid(indexOid)) + ereport(ERROR, + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("index \"%s\" for table \"%s\" does not exist", + indexname, RelationGetRelationName(rel))); + } + else + indexOid = InvalidOid; + + return indexOid; +} + +static const char * +RepackCommandAsString(RepackCommand cmd) +{ + switch (cmd) + { + case REPACK_COMMAND_REPACK: + return "REPACK"; + case REPACK_COMMAND_VACUUMFULL: + return "VACUUM"; + case REPACK_COMMAND_CLUSTER: + return "CLUSTER"; + } + return "???"; +} diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index ed03e3bd50d..62207ceff7e 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -348,7 +348,6 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) } } - /* * Sanity check DISABLE_PAGE_SKIPPING option. */ @@ -2280,8 +2279,9 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams params, if ((params.options & VACOPT_VERBOSE) != 0) cluster_params.options |= CLUOPT_VERBOSE; - /* VACUUM FULL is now a variant of CLUSTER; see cluster.c */ - cluster_rel(rel, InvalidOid, &cluster_params); + /* VACUUM FULL is a variant of REPACK; see cluster.c */ + cluster_rel(REPACK_COMMAND_VACUUMFULL, rel, InvalidOid, + &cluster_params); /* cluster_rel closes the relation, but keeps lock */ rel = NULL; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index a4b29c822e8..c463aa1415d 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -286,7 +286,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); AlterCompositeTypeStmt AlterUserMappingStmt AlterRoleStmt AlterRoleSetStmt AlterPolicyStmt AlterStatsStmt AlterDefaultPrivilegesStmt DefACLAction - AnalyzeStmt CallStmt ClosePortalStmt ClusterStmt CommentStmt + AnalyzeStmt CallStmt ClosePortalStmt CommentStmt ConstraintsSetStmt CopyStmt CreateAsStmt CreateCastStmt CreateDomainStmt CreateExtensionStmt CreateGroupStmt CreateOpClassStmt CreateOpFamilyStmt AlterOpFamilyStmt CreatePLangStmt @@ -303,7 +303,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); GrantStmt GrantRoleStmt ImportForeignSchemaStmt IndexStmt InsertStmt ListenStmt LoadStmt LockStmt MergeStmt NotifyStmt ExplainableStmt PreparableStmt CreateFunctionStmt AlterFunctionStmt ReindexStmt RemoveAggrStmt - RemoveFuncStmt RemoveOperStmt RenameStmt ReturnStmt RevokeStmt RevokeRoleStmt + RemoveFuncStmt RemoveOperStmt RenameStmt RepackStmt ReturnStmt RevokeStmt RevokeRoleStmt RuleActionStmt RuleActionStmtOrEmpty RuleStmt SecLabelStmt SelectStmt TransactionStmt TransactionStmtLegacy TruncateStmt UnlistenStmt UpdateStmt VacuumStmt @@ -322,7 +322,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type <str> opt_single_name %type <list> opt_qualified_name -%type <boolean> opt_concurrently +%type <boolean> opt_concurrently opt_usingindex %type <dbehavior> opt_drop_behavior %type <list> opt_utility_option_list %type <list> utility_option_list @@ -770,7 +770,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); QUOTE QUOTES RANGE READ REAL REASSIGN RECURSIVE REF_P REFERENCES REFERENCING - REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA + REFRESH REINDEX RELATIVE_P RELEASE RENAME REPACK REPEATABLE REPLACE REPLICA RESET RESPECT_P RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP ROUTINE ROUTINES ROW ROWS RULE @@ -1032,7 +1032,6 @@ stmt: | CallStmt | CheckPointStmt | ClosePortalStmt - | ClusterStmt | CommentStmt | ConstraintsSetStmt | CopyStmt @@ -1106,6 +1105,7 @@ stmt: | RemoveFuncStmt | RemoveOperStmt | RenameStmt + | RepackStmt | RevokeStmt | RevokeRoleStmt | RuleStmt @@ -1142,6 +1142,11 @@ opt_concurrently: | /*EMPTY*/ { $$ = false; } ; +opt_usingindex: + USING INDEX { $$ = true; } + | /* EMPTY */ { $$ = false; } + ; + opt_drop_behavior: CASCADE { $$ = DROP_CASCADE; } | RESTRICT { $$ = DROP_RESTRICT; } @@ -11959,38 +11964,93 @@ CreateConversionStmt: /***************************************************************************** * * QUERY: + * REPACK [ (options) ] [ <qualified_name> [ <name_list> ] [ USING INDEX <index_name> ] ] + * + * obsolete variants: * CLUSTER (options) [ <qualified_name> [ USING <index_name> ] ] * CLUSTER [VERBOSE] [ <qualified_name> [ USING <index_name> ] ] * CLUSTER [VERBOSE] <index_name> ON <qualified_name> (for pre-8.3) * *****************************************************************************/ -ClusterStmt: - CLUSTER '(' utility_option_list ')' qualified_name cluster_index_specification +RepackStmt: + REPACK opt_utility_option_list vacuum_relation USING INDEX name { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); - n->relation = $5; + n->command = REPACK_COMMAND_REPACK; + n->relation = (VacuumRelation *) $3; n->indexname = $6; + n->usingindex = true; + n->params = $2; + $$ = (Node *) n; + } + | REPACK opt_utility_option_list vacuum_relation opt_usingindex + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_REPACK; + n->relation = (VacuumRelation *) $3; + n->indexname = NULL; + n->usingindex = $4; + n->params = $2; + $$ = (Node *) n; + } + | REPACK '(' utility_option_list ')' + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_REPACK; + n->relation = NULL; + n->indexname = NULL; + n->usingindex = false; + n->params = $3; + $$ = (Node *) n; + } + | REPACK opt_usingindex + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_REPACK; + n->relation = NULL; + n->indexname = NULL; + n->usingindex = $2; + n->params = NIL; + $$ = (Node *) n; + } + | CLUSTER '(' utility_option_list ')' qualified_name cluster_index_specification + { + RepackStmt *n = makeNode(RepackStmt); + + n->command = REPACK_COMMAND_CLUSTER; + n->relation = makeNode(VacuumRelation); + n->relation->relation = $5; + n->indexname = $6; + n->usingindex = true; n->params = $3; $$ = (Node *) n; } | CLUSTER opt_utility_option_list { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); + n->command = REPACK_COMMAND_CLUSTER; n->relation = NULL; n->indexname = NULL; + n->usingindex = true; n->params = $2; $$ = (Node *) n; } /* unparenthesized VERBOSE kept for pre-14 compatibility */ | CLUSTER opt_verbose qualified_name cluster_index_specification { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); - n->relation = $3; + n->command = REPACK_COMMAND_CLUSTER; + n->relation = makeNode(VacuumRelation); + n->relation->relation = $3; n->indexname = $4; + n->usingindex = true; if ($2) n->params = list_make1(makeDefElem("verbose", NULL, @2)); $$ = (Node *) n; @@ -11998,20 +12058,25 @@ ClusterStmt: /* unparenthesized VERBOSE kept for pre-17 compatibility */ | CLUSTER VERBOSE { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); + n->command = REPACK_COMMAND_CLUSTER; n->relation = NULL; n->indexname = NULL; + n->usingindex = true; n->params = list_make1(makeDefElem("verbose", NULL, @2)); $$ = (Node *) n; } /* kept for pre-8.3 compatibility */ | CLUSTER opt_verbose name ON qualified_name { - ClusterStmt *n = makeNode(ClusterStmt); + RepackStmt *n = makeNode(RepackStmt); - n->relation = $5; + n->command = REPACK_COMMAND_CLUSTER; + n->relation = makeNode(VacuumRelation); + n->relation->relation = $5; n->indexname = $3; + n->usingindex = true; if ($2) n->params = list_make1(makeDefElem("verbose", NULL, @2)); $$ = (Node *) n; @@ -18028,6 +18093,7 @@ unreserved_keyword: | RELATIVE_P | RELEASE | RENAME + | REPACK | REPEATABLE | REPLACE | REPLICA @@ -18661,6 +18727,7 @@ bare_label_keyword: | RELATIVE_P | RELEASE | RENAME + | REPACK | REPEATABLE | REPLACE | REPLICA diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 918db53dd5e..1295dc25d02 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -277,9 +277,9 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) return COMMAND_OK_IN_RECOVERY | COMMAND_OK_IN_READ_ONLY_TXN; } - case T_ClusterStmt: case T_ReindexStmt: case T_VacuumStmt: + case T_RepackStmt: { /* * These commands write WAL, so they're not strictly @@ -854,14 +854,14 @@ standard_ProcessUtility(PlannedStmt *pstmt, ExecuteCallStmt(castNode(CallStmt, parsetree), params, isAtomicContext, dest); break; - case T_ClusterStmt: - cluster(pstate, (ClusterStmt *) parsetree, isTopLevel); - break; - case T_VacuumStmt: ExecVacuum(pstate, (VacuumStmt *) parsetree, isTopLevel); break; + case T_RepackStmt: + ExecRepack(pstate, (RepackStmt *) parsetree, isTopLevel); + break; + case T_ExplainStmt: ExplainQuery(pstate, (ExplainStmt *) parsetree, params, dest); break; @@ -2850,10 +2850,6 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_CALL; break; - case T_ClusterStmt: - tag = CMDTAG_CLUSTER; - break; - case T_VacuumStmt: if (((VacuumStmt *) parsetree)->is_vacuumcmd) tag = CMDTAG_VACUUM; @@ -2861,6 +2857,10 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_ANALYZE; break; + case T_RepackStmt: + tag = CMDTAG_REPACK; + break; + case T_ExplainStmt: tag = CMDTAG_EXPLAIN; break; @@ -3498,7 +3498,7 @@ GetCommandLogLevel(Node *parsetree) lev = LOGSTMT_ALL; break; - case T_ClusterStmt: + case T_RepackStmt: lev = LOGSTMT_DDL; break; diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index a710508979e..1a7bb1e4a31 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -289,6 +289,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS) cmdtype = PROGRESS_COMMAND_ANALYZE; else if (pg_strcasecmp(cmd, "CLUSTER") == 0) cmdtype = PROGRESS_COMMAND_CLUSTER; + else if (pg_strcasecmp(cmd, "REPACK") == 0) + cmdtype = PROGRESS_COMMAND_REPACK; else if (pg_strcasecmp(cmd, "CREATE INDEX") == 0) cmdtype = PROGRESS_COMMAND_CREATE_INDEX; else if (pg_strcasecmp(cmd, "BASEBACKUP") == 0) diff --git a/src/bin/psql/tab-complete.in.c b/src/bin/psql/tab-complete.in.c index 36ea6a4d557..b1fe7703296 100644 --- a/src/bin/psql/tab-complete.in.c +++ b/src/bin/psql/tab-complete.in.c @@ -1257,7 +1257,7 @@ static const char *const sql_commands[] = { "DELETE FROM", "DISCARD", "DO", "DROP", "END", "EXECUTE", "EXPLAIN", "FETCH", "GRANT", "IMPORT FOREIGN SCHEMA", "INSERT INTO", "LISTEN", "LOAD", "LOCK", "MERGE INTO", "MOVE", "NOTIFY", "PREPARE", - "REASSIGN", "REFRESH MATERIALIZED VIEW", "REINDEX", "RELEASE", + "REASSIGN", "REFRESH MATERIALIZED VIEW", "REINDEX", "RELEASE", "REPACK", "RESET", "REVOKE", "ROLLBACK", "SAVEPOINT", "SECURITY LABEL", "SELECT", "SET", "SHOW", "START", "TABLE", "TRUNCATE", "UNLISTEN", "UPDATE", "VACUUM", "VALUES", "WITH", @@ -5008,6 +5008,37 @@ match_previous_words(int pattern_id, COMPLETE_WITH_QUERY(Query_for_list_of_tablespaces); } +/* REPACK */ + else if (Matches("REPACK")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_clusterables); + else if (Matches("REPACK", "(*)")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_clusterables); + /* If we have REPACK <sth>, then add "USING INDEX" */ + else if (Matches("REPACK", MatchAnyExcept("("))) + COMPLETE_WITH("USING INDEX"); + /* If we have REPACK (*) <sth>, then add "USING INDEX" */ + else if (Matches("REPACK", "(*)", MatchAny)) + COMPLETE_WITH("USING INDEX"); + /* If we have REPACK <sth> USING, then add the index as well */ + else if (Matches("REPACK", MatchAny, "USING", "INDEX")) + { + set_completion_reference(prev3_wd); + COMPLETE_WITH_SCHEMA_QUERY(Query_for_index_of_table); + } + else if (HeadMatches("REPACK", "(*") && + !HeadMatches("REPACK", "(*)")) + { + /* + * This fires if we're in an unfinished parenthesized option list. + * get_previous_words treats a completed parenthesized option list as + * one word, so the above test is correct. + */ + if (ends_with(prev_wd, '(') || ends_with(prev_wd, ',')) + COMPLETE_WITH("VERBOSE"); + else if (TailMatches("VERBOSE")) + COMPLETE_WITH("ON", "OFF"); + } + /* SECURITY LABEL */ else if (Matches("SECURITY")) COMPLETE_WITH("LABEL"); diff --git a/src/bin/scripts/Makefile b/src/bin/scripts/Makefile index 019ca06455d..f0c1bd4175c 100644 --- a/src/bin/scripts/Makefile +++ b/src/bin/scripts/Makefile @@ -16,7 +16,7 @@ subdir = src/bin/scripts top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -PROGRAMS = createdb createuser dropdb dropuser clusterdb vacuumdb reindexdb pg_isready +PROGRAMS = createdb createuser dropdb dropuser clusterdb vacuumdb reindexdb pg_isready pg_repackdb override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) @@ -31,6 +31,7 @@ clusterdb: clusterdb.o common.o $(WIN32RES) | submake-libpq submake-libpgport su vacuumdb: vacuumdb.o vacuuming.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils reindexdb: reindexdb.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils pg_isready: pg_isready.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils +pg_repackdb: pg_repackdb.o vacuuming.o common.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) createdb$(X) '$(DESTDIR)$(bindir)'/createdb$(X) @@ -41,6 +42,7 @@ install: all installdirs $(INSTALL_PROGRAM) vacuumdb$(X) '$(DESTDIR)$(bindir)'/vacuumdb$(X) $(INSTALL_PROGRAM) reindexdb$(X) '$(DESTDIR)$(bindir)'/reindexdb$(X) $(INSTALL_PROGRAM) pg_isready$(X) '$(DESTDIR)$(bindir)'/pg_isready$(X) + $(INSTALL_PROGRAM) pg_repackdb$(X) '$(DESTDIR)$(bindir)'/pg_repackdb$(X) installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' diff --git a/src/bin/scripts/meson.build b/src/bin/scripts/meson.build index a4fed59d1c9..be573cae682 100644 --- a/src/bin/scripts/meson.build +++ b/src/bin/scripts/meson.build @@ -42,6 +42,7 @@ vacuuming_common = static_library('libvacuuming_common', binaries = [ 'vacuumdb', + 'pg_repackdb', ] foreach binary : binaries binary_sources = files('@[email protected]'.format(binary)) @@ -80,6 +81,7 @@ tests += { 't/100_vacuumdb.pl', 't/101_vacuumdb_all.pl', 't/102_vacuumdb_stages.pl', + 't/103_repackdb.pl', 't/200_connstr.pl', ], }, diff --git a/src/bin/scripts/pg_repackdb.c b/src/bin/scripts/pg_repackdb.c new file mode 100644 index 00000000000..1edfa34ed0f --- /dev/null +++ b/src/bin/scripts/pg_repackdb.c @@ -0,0 +1,242 @@ +/*------------------------------------------------------------------------- + * + * pg_repackdb + * An utility to run REPACK + * + * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * FIXME: this is missing a way to specify the index to use to repack one + * table, or whether to pass a WITH INDEX clause when multiple tables are + * used. Something like --index[=indexname]. Adding that bleeds into + * vacuuming.c as well. + * + * src/bin/scripts/pg_repackdb.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" + +#include <limits.h> + +#include "common.h" +#include "common/logging.h" +#include "fe_utils/option_utils.h" +#include "vacuuming.h" + +static void help(const char *progname); +static void check_objfilter(bits32 objfilter); + +int +main(int argc, char *argv[]) +{ + static struct option long_options[] = { + {"host", required_argument, NULL, 'h'}, + {"port", required_argument, NULL, 'p'}, + {"username", required_argument, NULL, 'U'}, + {"no-password", no_argument, NULL, 'w'}, + {"password", no_argument, NULL, 'W'}, + {"echo", no_argument, NULL, 'e'}, + {"quiet", no_argument, NULL, 'q'}, + {"dbname", required_argument, NULL, 'd'}, + {"analyze", no_argument, NULL, 'z'}, + {"all", no_argument, NULL, 'a'}, + /* XXX this could be 'i', but optional_arg is messy */ + {"index", optional_argument, NULL, 1}, + {"table", required_argument, NULL, 't'}, + {"verbose", no_argument, NULL, 'v'}, + {"jobs", required_argument, NULL, 'j'}, + {"schema", required_argument, NULL, 'n'}, + {"exclude-schema", required_argument, NULL, 'N'}, + {"maintenance-db", required_argument, NULL, 2}, + {NULL, 0, NULL, 0} + }; + + const char *progname; + int optindex; + int c; + const char *dbname = NULL; + const char *maintenance_db = NULL; + ConnParams cparams; + bool echo = false; + bool quiet = false; + vacuumingOptions vacopts; + SimpleStringList objects = {NULL, NULL}; + int concurrentCons = 1; + int tbl_count = 0; + int ret; + + /* initialize options */ + memset(&vacopts, 0, sizeof(vacopts)); + vacopts.mode = MODE_REPACK; + + /* the same for connection parameters */ + memset(&cparams, 0, sizeof(cparams)); + cparams.prompt_password = TRI_DEFAULT; + + pg_logging_init(argv[0]); + progname = get_progname(argv[0]); + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pgscripts")); + + handle_help_version_opts(argc, argv, progname, help); + + while ((c = getopt_long(argc, argv, "ad:eh:j:n:N:p:qt:U:vwWz", + long_options, &optindex)) != -1) + { + switch (c) + { + case 'a': + vacopts.objfilter |= OBJFILTER_ALL_DBS; + break; + case 'd': + vacopts.objfilter |= OBJFILTER_DATABASE; + dbname = pg_strdup(optarg); + break; + case 'e': + echo = true; + break; + case 'h': + cparams.pghost = pg_strdup(optarg); + break; + case 'j': + if (!option_parse_int(optarg, "-j/--jobs", 1, INT_MAX, + &concurrentCons)) + exit(1); + break; + case 'n': + vacopts.objfilter |= OBJFILTER_SCHEMA; + simple_string_list_append(&objects, optarg); + break; + case 'N': + vacopts.objfilter |= OBJFILTER_SCHEMA_EXCLUDE; + simple_string_list_append(&objects, optarg); + break; + case 'p': + cparams.pgport = pg_strdup(optarg); + break; + case 'q': + quiet = true; + break; + case 't': + vacopts.objfilter |= OBJFILTER_TABLE; + simple_string_list_append(&objects, optarg); + tbl_count++; + break; + case 'U': + cparams.pguser = pg_strdup(optarg); + break; + case 'v': + vacopts.verbose = true; + break; + case 'w': + cparams.prompt_password = TRI_NO; + break; + case 'W': + cparams.prompt_password = TRI_YES; + break; + case 'z': + vacopts.and_analyze = true; + break; + case 1: + vacopts.using_index = true; + if (optarg) + vacopts.indexname = pg_strdup(optarg); + else + vacopts.indexname = NULL; + break; + case 2: + maintenance_db = pg_strdup(optarg); + break; + default: + /* getopt_long already emitted a complaint */ + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + } + + /* + * Non-option argument specifies database name as long as it wasn't + * already specified with -d / --dbname + */ + if (optind < argc && dbname == NULL) + { + vacopts.objfilter |= OBJFILTER_DATABASE; + dbname = argv[optind]; + optind++; + } + + if (optind < argc) + { + pg_log_error("too many command-line arguments (first is \"%s\")", + argv[optind]); + pg_log_error_hint("Try \"%s --help\" for more information.", progname); + exit(1); + } + + /* + * Validate the combination of filters specified in the command-line + * options. + */ + check_objfilter(vacopts.objfilter); + + ret = vacuuming_main(&cparams, dbname, maintenance_db, &vacopts, + &objects, tbl_count, concurrentCons, + progname, echo, quiet); + exit(ret); +} + +/* + * Verify that the filters used at command line are compatible. + */ +void +check_objfilter(bits32 objfilter) +{ + if ((objfilter & OBJFILTER_ALL_DBS) && + (objfilter & OBJFILTER_DATABASE)) + pg_fatal("cannot repack all databases and a specific one at the same time"); + + if ((objfilter & OBJFILTER_TABLE) && + (objfilter & OBJFILTER_SCHEMA)) + pg_fatal("cannot repack all tables in schema(s) and specific table(s) at the same time"); + + if ((objfilter & OBJFILTER_TABLE) && + (objfilter & OBJFILTER_SCHEMA_EXCLUDE)) + pg_fatal("cannot repack specific table(s) and exclude schema(s) at the same time"); + + if ((objfilter & OBJFILTER_SCHEMA) && + (objfilter & OBJFILTER_SCHEMA_EXCLUDE)) + pg_fatal("cannot repack all tables in schema(s) and exclude schema(s) at the same time"); +} + +static void +help(const char *progname) +{ + printf(_("%s repacks a PostgreSQL database.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]... [DBNAME]\n"), progname); + printf(_("\nOptions:\n")); + printf(_(" -a, --all repack all databases\n")); + printf(_(" -d, --dbname=DBNAME database to repack\n")); + printf(_(" -e, --echo show the commands being sent to the server\n")); + printf(_(" --index[=INDEX] repack following an index\n")); + printf(_(" -j, --jobs=NUM use this many concurrent connections to repack\n")); + printf(_(" -n, --schema=SCHEMA repack tables in the specified schema(s) only\n")); + printf(_(" -N, --exclude-schema=SCHEMA do not repack tables in the specified schema(s)\n")); + printf(_(" -q, --quiet don't write any messages\n")); + printf(_(" -t, --table='TABLE[(COLUMNS)]' repack specific table(s) only\n")); + printf(_(" -v, --verbose write a lot of output\n")); + printf(_(" -V, --version output version information, then exit\n")); + printf(_(" -z, --analyze update optimizer statistics\n")); + printf(_(" -?, --help show this help, then exit\n")); + printf(_("\nConnection options:\n")); + printf(_(" -h, --host=HOSTNAME database server host or socket directory\n")); + printf(_(" -p, --port=PORT database server port\n")); + printf(_(" -U, --username=USERNAME user name to connect as\n")); + printf(_(" -w, --no-password never prompt for password\n")); + printf(_(" -W, --password force password prompt\n")); + printf(_(" --maintenance-db=DBNAME alternate maintenance database\n")); + printf(_("\nRead the description of the SQL command REPACK for details.\n")); + printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT); + printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); +} diff --git a/src/bin/scripts/t/103_repackdb.pl b/src/bin/scripts/t/103_repackdb.pl new file mode 100644 index 00000000000..cadce9b837c --- /dev/null +++ b/src/bin/scripts/t/103_repackdb.pl @@ -0,0 +1,47 @@ +# Copyright (c) 2021-2025, PostgreSQL Global Development Group + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +program_help_ok('pg_repackdb'); +program_version_ok('pg_repackdb'); +program_options_handling_ok('pg_repackdb'); + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->start; + +$node->issues_sql_like( + [ 'pg_repackdb', 'postgres', '-t', 'pg_class'], + qr/statement: REPACK.*pg_class;/, + 'pg_repackdb processes a single table'); + +$node->safe_psql('postgres', 'CREATE USER testusr; + GRANT CREATE ON SCHEMA public TO testusr'); +$node->safe_psql('postgres', + 'CREATE TABLE cluster_1 (a int primary key); + ALTER TABLE cluster_1 CLUSTER ON cluster_1_pkey; + CREATE TABLE cluster_2 (a int unique); + ALTER TABLE cluster_2 CLUSTER ON cluster_2_a_key;', + extra_params => ['-U' => 'testusr']); + +$node->issues_sql_like( + [ 'pg_repackdb', 'postgres', '-U', 'testusr' ], + qr/statement: REPACK.*;/, + 'SQL REPACK run'); + +$node->issues_sql_like( + [ 'pg_repackdb', 'postgres', '--index'], + qr/statement: REPACK.*cluster_1 USING INDEX.*statement: REPACK.*cluster_2 USING INDEX/ms, + 'pg_repackdb --index chooses multiple tables'); + +$node->issues_sql_like( + [ 'pg_repackdb', 'postgres', '--analyze', '-t', 'cluster_1'], + qr/statement: REPACK \(ANALYZE\) public.cluster_1/, + 'pg_repackdb --analyze works'); + +done_testing(); diff --git a/src/bin/scripts/vacuuming.c b/src/bin/scripts/vacuuming.c index f836f21fb03..d8d77cabe43 100644 --- a/src/bin/scripts/vacuuming.c +++ b/src/bin/scripts/vacuuming.c @@ -1,6 +1,6 @@ /*------------------------------------------------------------------------- * vacuuming.c - * Helper routines for vacuumdb + * Helper routines for vacuumdb and pg_repackdb * * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -43,8 +43,8 @@ static SimpleStringList *retrieve_objects(PGconn *conn, static void free_retrieved_objects(SimpleStringList *list); static void prepare_vacuum_command(PGconn *conn, PQExpBuffer sql, vacuumingOptions *vacopts, const char *table); -static void run_vacuum_command(PGconn *conn, const char *sql, bool echo, - const char *table); +static void run_vacuum_command(PGconn *conn, vacuumingOptions *vacopts, + const char *sql, bool echo, const char *table); /* * Executes vacuum/analyze as indicated. Returns 0 if the plan is carried @@ -194,6 +194,14 @@ vacuum_one_database(ConnParams *cparams, conn = connectDatabase(cparams, progname, echo, false, true); + if (vacopts->mode == MODE_REPACK && PQserverVersion(conn) < 190000) + { + /* XXX arguably, here we should use VACUUM FULL instead of failing */ + PQfinish(conn); + pg_fatal("cannot use the \"%s\" command on server versions older than PostgreSQL %s", + "REPACK", "19"); + } + if (vacopts->disable_page_skipping && PQserverVersion(conn) < 90600) { PQfinish(conn); @@ -286,9 +294,18 @@ vacuum_one_database(ConnParams *cparams, if (vacopts->mode == MODE_ANALYZE_IN_STAGES) printf(_("%s: processing database \"%s\": %s\n"), progname, PQdb(conn), _(stage_messages[stage])); - else + else if (vacopts->mode == MODE_ANALYZE) + printf(_("%s: analyzing database \"%s\"\n"), + progname, PQdb(conn)); + else if (vacopts->mode == MODE_VACUUM) printf(_("%s: vacuuming database \"%s\"\n"), progname, PQdb(conn)); + else + { + Assert(vacopts->mode == MODE_REPACK); + printf(_("%s: repacking database \"%s\"\n"), + progname, PQdb(conn)); + } fflush(stdout); } @@ -383,7 +400,7 @@ vacuum_one_database(ConnParams *cparams, * through ParallelSlotsGetIdle. */ ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL); - run_vacuum_command(free_slot->connection, sql.data, + run_vacuum_command(free_slot->connection, vacopts, sql.data, echo, tabname); cell = cell->next; @@ -408,7 +425,7 @@ vacuum_one_database(ConnParams *cparams, } ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL); - run_vacuum_command(free_slot->connection, cmd, echo, NULL); + run_vacuum_command(free_slot->connection, vacopts, cmd, echo, NULL); if (!ParallelSlotsWaitCompletion(sa)) ret = EXIT_FAILURE; /* error already reported by handler */ @@ -636,6 +653,35 @@ retrieve_objects(PGconn *conn, vacuumingOptions *vacopts, " AND listed_objects.object_oid IS NOT NULL\n"); } + /* + * In REPACK mode, if the 'using_index' option was given but no index + * name, filter only tables that have an index with indisclustered set. + * (If an index name is given, we trust the user to pass a reasonable list + * of tables.) + * + * XXX it may be worth printing an error if an index name is given with no + * list of tables. + */ + if (vacopts->mode == MODE_REPACK && + vacopts->using_index && !vacopts->indexname) + { + appendPQExpBufferStr(&catalog_query, + " AND EXISTS (SELECT 1 FROM pg_catalog.pg_index\n" + " WHERE indrelid = c.oid AND indisclustered)\n"); + } + + /* + * In REPACK mode, only consider the tables that the current user has + * MAINTAIN privileges on. XXX maybe we should do this in all cases, not + * just REPACK. The vacuumdb output is too noisy for no reason. + */ + if (vacopts->mode == MODE_REPACK) + { + appendPQExpBufferStr(&catalog_query, + " AND pg_catalog.has_table_privilege(current_user, " + "c.oid, 'MAINTAIN')\n"); + } + /* * If no tables were listed, filter for the relevant relation types. If * tables were given via --table, don't bother filtering by relation type. @@ -874,8 +920,10 @@ prepare_vacuum_command(PGconn *conn, PQExpBuffer sql, if (vacopts->verbose) appendPQExpBufferStr(sql, " VERBOSE"); } + + appendPQExpBuffer(sql, " %s", table); } - else + else if (vacopts->mode == MODE_VACUUM) { appendPQExpBufferStr(sql, "VACUUM"); @@ -989,9 +1037,39 @@ prepare_vacuum_command(PGconn *conn, PQExpBuffer sql, if (vacopts->and_analyze) appendPQExpBufferStr(sql, " ANALYZE"); } + + appendPQExpBuffer(sql, " %s", table); + } + else if (vacopts->mode == MODE_REPACK) + { + appendPQExpBufferStr(sql, "REPACK"); + + if (vacopts->verbose) + { + appendPQExpBuffer(sql, "%sVERBOSE", sep); + sep = comma; + } + if (vacopts->and_analyze) + { + appendPQExpBuffer(sql, "%sANALYZE", sep); + sep = comma; + } + + if (sep != paren) + appendPQExpBufferChar(sql, ')'); + + appendPQExpBuffer(sql, " %s", table); + + if (vacopts->using_index) + { + appendPQExpBuffer(sql, " USING INDEX"); + if (vacopts->indexname) + appendPQExpBuffer(sql, " %s", fmtIdEnc(vacopts->indexname, + PQclientEncoding(conn))); + } } - appendPQExpBuffer(sql, " %s;", table); + appendPQExpBufferChar(sql, ';'); } /* @@ -1001,8 +1079,8 @@ prepare_vacuum_command(PGconn *conn, PQExpBuffer sql, * Any errors during command execution are reported to stderr. */ static void -run_vacuum_command(PGconn *conn, const char *sql, bool echo, - const char *table) +run_vacuum_command(PGconn *conn, vacuumingOptions *vacopts, + const char *sql, bool echo, const char *table) { bool status; @@ -1015,13 +1093,21 @@ run_vacuum_command(PGconn *conn, const char *sql, bool echo, { if (table) { - pg_log_error("vacuuming of table \"%s\" in database \"%s\" failed: %s", - table, PQdb(conn), PQerrorMessage(conn)); + if (vacopts->mode == MODE_VACUUM) + pg_log_error("vacuuming of table \"%s\" in database \"%s\" failed: %s", + table, PQdb(conn), PQerrorMessage(conn)); + else + pg_log_error("repacking of table \"%s\" in database \"%s\" failed: %s", + table, PQdb(conn), PQerrorMessage(conn)); } else { - pg_log_error("vacuuming of database \"%s\" failed: %s", - PQdb(conn), PQerrorMessage(conn)); + if (vacopts->mode == MODE_VACUUM) + pg_log_error("vacuuming of database \"%s\" failed: %s", + PQdb(conn), PQerrorMessage(conn)); + else + pg_log_error("repacking of database \"%s\" failed: %s", + PQdb(conn), PQerrorMessage(conn)); } } } diff --git a/src/bin/scripts/vacuuming.h b/src/bin/scripts/vacuuming.h index 49f968b32e5..665dbaedfad 100644 --- a/src/bin/scripts/vacuuming.h +++ b/src/bin/scripts/vacuuming.h @@ -20,6 +20,7 @@ typedef enum { MODE_VACUUM, + MODE_REPACK, MODE_ANALYZE, MODE_ANALYZE_IN_STAGES } RunMode; @@ -37,6 +38,8 @@ typedef struct vacuumingOptions bool and_analyze; bool full; bool freeze; + bool using_index; + char *indexname; bool disable_page_skipping; bool skip_locked; int min_xid_age; diff --git a/src/include/commands/cluster.h b/src/include/commands/cluster.h index 60088a64cbb..652542e8e65 100644 --- a/src/include/commands/cluster.h +++ b/src/include/commands/cluster.h @@ -24,6 +24,7 @@ #define CLUOPT_RECHECK 0x02 /* recheck relation state */ #define CLUOPT_RECHECK_ISCLUSTERED 0x04 /* recheck relation state for * indisclustered */ +#define CLUOPT_ANALYZE 0x08 /* do an ANALYZE */ /* options for CLUSTER */ typedef struct ClusterParams @@ -31,8 +32,11 @@ typedef struct ClusterParams bits32 options; /* bitmask of CLUOPT_* */ } ClusterParams; -extern void cluster(ParseState *pstate, ClusterStmt *stmt, bool isTopLevel); -extern void cluster_rel(Relation OldHeap, Oid indexOid, ClusterParams *params); + +extern void ExecRepack(ParseState *pstate, RepackStmt *stmt, bool isTopLevel); + +extern void cluster_rel(RepackCommand command, Relation OldHeap, Oid indexOid, + ClusterParams *params); extern void check_index_is_clusterable(Relation OldHeap, Oid indexOid, LOCKMODE lockmode); extern void mark_index_clustered(Relation rel, Oid indexOid, bool is_internal); diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index 1cde4bd9bcf..ebf004b7aa5 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -56,28 +56,34 @@ #define PROGRESS_ANALYZE_PHASE_COMPUTE_EXT_STATS 4 #define PROGRESS_ANALYZE_PHASE_FINALIZE_ANALYZE 5 -/* Progress parameters for cluster */ -#define PROGRESS_CLUSTER_COMMAND 0 -#define PROGRESS_CLUSTER_PHASE 1 -#define PROGRESS_CLUSTER_INDEX_RELID 2 -#define PROGRESS_CLUSTER_HEAP_TUPLES_SCANNED 3 -#define PROGRESS_CLUSTER_HEAP_TUPLES_WRITTEN 4 -#define PROGRESS_CLUSTER_TOTAL_HEAP_BLKS 5 -#define PROGRESS_CLUSTER_HEAP_BLKS_SCANNED 6 -#define PROGRESS_CLUSTER_INDEX_REBUILD_COUNT 7 +/* + * Progress parameters for REPACK. + * + * Values for PROGRESS_REPACK_COMMAND are defined as in RepackCommand. + * + * Note: Since REPACK shares code with CLUSTER, these values are also + * used by CLUSTER. (CLUSTER is now deprecated, so it makes little sense to + * introduce a separate set of constants.) + */ +#define PROGRESS_REPACK_COMMAND 0 +#define PROGRESS_REPACK_PHASE 1 +#define PROGRESS_REPACK_INDEX_RELID 2 +#define PROGRESS_REPACK_HEAP_TUPLES_SCANNED 3 +#define PROGRESS_REPACK_HEAP_TUPLES_WRITTEN 4 +#define PROGRESS_REPACK_TOTAL_HEAP_BLKS 5 +#define PROGRESS_REPACK_HEAP_BLKS_SCANNED 6 +#define PROGRESS_REPACK_INDEX_REBUILD_COUNT 7 -/* Phases of cluster (as advertised via PROGRESS_CLUSTER_PHASE) */ -#define PROGRESS_CLUSTER_PHASE_SEQ_SCAN_HEAP 1 -#define PROGRESS_CLUSTER_PHASE_INDEX_SCAN_HEAP 2 -#define PROGRESS_CLUSTER_PHASE_SORT_TUPLES 3 -#define PROGRESS_CLUSTER_PHASE_WRITE_NEW_HEAP 4 -#define PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES 5 -#define PROGRESS_CLUSTER_PHASE_REBUILD_INDEX 6 -#define PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP 7 - -/* Commands of PROGRESS_CLUSTER */ -#define PROGRESS_CLUSTER_COMMAND_CLUSTER 1 -#define PROGRESS_CLUSTER_COMMAND_VACUUM_FULL 2 +/* + * Phases of repack (as advertised via PROGRESS_REPACK_PHASE). + */ +#define PROGRESS_REPACK_PHASE_SEQ_SCAN_HEAP 1 +#define PROGRESS_REPACK_PHASE_INDEX_SCAN_HEAP 2 +#define PROGRESS_REPACK_PHASE_SORT_TUPLES 3 +#define PROGRESS_REPACK_PHASE_WRITE_NEW_HEAP 4 +#define PROGRESS_REPACK_PHASE_SWAP_REL_FILES 5 +#define PROGRESS_REPACK_PHASE_REBUILD_INDEX 6 +#define PROGRESS_REPACK_PHASE_FINAL_CLEANUP 7 /* Progress parameters for CREATE INDEX */ /* 3, 4 and 5 reserved for "waitfor" metrics */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index ecbddd12e1b..525f6e6d6a5 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3951,18 +3951,6 @@ typedef struct AlterSystemStmt VariableSetStmt *setstmt; /* SET subcommand */ } AlterSystemStmt; -/* ---------------------- - * Cluster Statement (support pbrown's cluster index implementation) - * ---------------------- - */ -typedef struct ClusterStmt -{ - NodeTag type; - RangeVar *relation; /* relation being indexed, or NULL if all */ - char *indexname; /* original index defined */ - List *params; /* list of DefElem nodes */ -} ClusterStmt; - /* ---------------------- * Vacuum and Analyze Statements * @@ -3975,7 +3963,7 @@ typedef struct VacuumStmt NodeTag type; List *options; /* list of DefElem nodes */ List *rels; /* list of VacuumRelation, or NIL for all */ - bool is_vacuumcmd; /* true for VACUUM, false for ANALYZE */ + bool is_vacuumcmd; /* true for VACUUM, false otherwise */ } VacuumStmt; /* @@ -3993,6 +3981,27 @@ typedef struct VacuumRelation List *va_cols; /* list of column names, or NIL for all */ } VacuumRelation; +/* ---------------------- + * Repack Statement + * ---------------------- + */ +typedef enum RepackCommand +{ + REPACK_COMMAND_CLUSTER = 1, + REPACK_COMMAND_REPACK, + REPACK_COMMAND_VACUUMFULL, +} RepackCommand; + +typedef struct RepackStmt +{ + NodeTag type; + RepackCommand command; /* type of command being run */ + VacuumRelation *relation; /* relation being repacked */ + char *indexname; /* order tuples by this index */ + bool usingindex; /* whether USING INDEX is specified */ + List *params; /* list of DefElem nodes */ +} RepackStmt; + /* ---------------------- * Explain Statement * diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 84182eaaae2..87f6c226c43 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -375,6 +375,7 @@ PG_KEYWORD("reindex", REINDEX, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("relative", RELATIVE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("release", RELEASE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("rename", RENAME, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("repack", REPACK, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("repeatable", REPEATABLE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replace", REPLACE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("replica", REPLICA, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index d250a714d59..cceb312f2b3 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -196,6 +196,7 @@ PG_CMDTAG(CMDTAG_REASSIGN_OWNED, "REASSIGN OWNED", false, false, false) PG_CMDTAG(CMDTAG_REFRESH_MATERIALIZED_VIEW, "REFRESH MATERIALIZED VIEW", true, false, false) PG_CMDTAG(CMDTAG_REINDEX, "REINDEX", true, false, false) PG_CMDTAG(CMDTAG_RELEASE, "RELEASE", false, false, false) +PG_CMDTAG(CMDTAG_REPACK, "REPACK", false, false, false) PG_CMDTAG(CMDTAG_RESET, "RESET", false, false, false) PG_CMDTAG(CMDTAG_REVOKE, "REVOKE", true, false, false) PG_CMDTAG(CMDTAG_REVOKE_ROLE, "REVOKE ROLE", false, false, false) diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h index dda813ab407..e69e366dcdc 100644 --- a/src/include/utils/backend_progress.h +++ b/src/include/utils/backend_progress.h @@ -28,6 +28,7 @@ typedef enum ProgressCommandType PROGRESS_COMMAND_CREATE_INDEX, PROGRESS_COMMAND_BASEBACKUP, PROGRESS_COMMAND_COPY, + PROGRESS_COMMAND_REPACK, } ProgressCommandType; #define PGSTAT_NUM_PROGRESS_PARAM 20 diff --git a/src/test/regress/expected/cluster.out b/src/test/regress/expected/cluster.out index 4d40a6809ab..277854418fa 100644 --- a/src/test/regress/expected/cluster.out +++ b/src/test/regress/expected/cluster.out @@ -495,6 +495,43 @@ ALTER TABLE clstrpart SET WITHOUT CLUSTER; ERROR: cannot mark index clustered in partitioned table ALTER TABLE clstrpart CLUSTER ON clstrpart_idx; ERROR: cannot mark index clustered in partitioned table +-- Check that REPACK sets new relfilenodes: it should process exactly the same +-- tables as CLUSTER did. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart USING INDEX clstrpart_idx; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + relname | level | relkind | ?column? +-------------+-------+---------+---------- + clstrpart | 0 | p | t + clstrpart1 | 1 | p | t + clstrpart11 | 2 | r | f + clstrpart12 | 2 | p | t + clstrpart2 | 1 | r | f + clstrpart3 | 1 | p | t + clstrpart33 | 2 | r | f +(7 rows) + +-- And finally the same for REPACK w/o index. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + relname | level | relkind | ?column? +-------------+-------+---------+---------- + clstrpart | 0 | p | t + clstrpart1 | 1 | p | t + clstrpart11 | 2 | r | f + clstrpart12 | 2 | p | t + clstrpart2 | 1 | r | f + clstrpart3 | 1 | p | t + clstrpart33 | 2 | r | f +(7 rows) + DROP TABLE clstrpart; -- Ownership of partitions is checked CREATE TABLE ptnowner(i int unique) PARTITION BY LIST (i); @@ -513,7 +550,7 @@ CREATE TEMP TABLE ptnowner_oldnodes AS JOIN pg_class AS c ON c.oid=tree.relid; SET SESSION AUTHORIZATION regress_ptnowner; CLUSTER ptnowner USING ptnowner_i_idx; -WARNING: permission denied to cluster "ptnowner2", skipping it +WARNING: permission denied to execute CLUSTER on "ptnowner2", skipping it RESET SESSION AUTHORIZATION; SELECT a.relname, a.relfilenode=b.relfilenode FROM pg_class a JOIN ptnowner_oldnodes b USING (oid) ORDER BY a.relname COLLATE "C"; @@ -665,6 +702,101 @@ SELECT * FROM clstr_expression WHERE -a = -3 ORDER BY -a, b; (4 rows) COMMIT; +---------------------------------------------------------------------- +-- +-- REPACK +-- +---------------------------------------------------------------------- +-- REPACK handles individual tables identically to CLUSTER, but it's worth +-- checking if it handles table hierarchies identically as well. +REPACK clstr_tst USING INDEX clstr_tst_c; +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 2'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + a | b | c | substring | length +----+-----+------------------+--------------------------------+-------- + 10 | 14 | catorce | | + 18 | 5 | cinco | | + 9 | 4 | cuatro | | + 26 | 19 | diecinueve | | + 12 | 18 | dieciocho | | + 30 | 16 | dieciseis | | + 24 | 17 | diecisiete | | + 2 | 10 | diez | | + 23 | 12 | doce | | + 11 | 2 | dos | | + 25 | 9 | nueve | | + 31 | 8 | ocho | | + 1 | 11 | once | | + 28 | 15 | quince | | + 32 | 6 | seis | xyzzyxyzzyxyzzyxyzzyxyzzyxyzzy | 500000 + 29 | 7 | siete | | + 15 | 13 | trece | | + 22 | 30 | treinta | | + 17 | 32 | treinta y dos | | + 3 | 31 | treinta y uno | | + 5 | 3 | tres | | + 20 | 1 | uno | | + 6 | 20 | veinte | | + 14 | 25 | veinticinco | | + 21 | 24 | veinticuatro | | + 4 | 22 | veintidos | | + 19 | 29 | veintinueve | | + 16 | 28 | veintiocho | | + 27 | 26 | veintiseis | | + 13 | 27 | veintisiete | | + 7 | 23 | veintitres | | + 8 | 21 | veintiuno | | + 0 | 100 | in child table | | + 0 | 100 | in child table 2 | | +(34 rows) + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); +ERROR: insert or update on table "clstr_tst" violates foreign key constraint "clstr_tst_con" +DETAIL: Key (b)=(1111) is not present in table "clstr_tst_s". +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; + conname +---------------------- + clstr_tst_a_not_null + clstr_tst_con + clstr_tst_pkey +(3 rows) + +-- Verify partial analyze works +REPACK (ANALYZE) clstr_tst (a); +REPACK (ANALYZE) clstr_tst; +REPACK (VERBOSE) clstr_tst (a); +ERROR: ANALYZE option must be specified when a column list is provided +-- REPACK w/o argument performs no ordering, so we can only check which tables +-- have the relfilenode changed. +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_old AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); +SET SESSION AUTHORIZATION regress_clstr_user; +SET client_min_messages = ERROR; -- order of "skipping" warnings may vary +REPACK; +RESET client_min_messages; +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_new AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); +-- Do the actual comparison. Unlike CLUSTER, clstr_3 should have been +-- processed because there is nothing like clustering index here. +SELECT o.relname FROM relnodes_old o +JOIN relnodes_new n ON o.relname = n.relname +WHERE o.relfilenode <> n.relfilenode +ORDER BY o.relname; + relname +--------- + clstr_1 + clstr_3 +(2 rows) + -- clean up DROP TABLE clustertest; DROP TABLE clstr_1; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 77e25ca029e..bd872ebd13a 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1994,34 +1994,23 @@ pg_stat_progress_basebackup| SELECT pid, ELSE NULL::text END AS backup_type FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20); -pg_stat_progress_cluster| SELECT s.pid, - s.datid, - d.datname, - s.relid, - CASE s.param1 - WHEN 1 THEN 'CLUSTER'::text - WHEN 2 THEN 'VACUUM FULL'::text - ELSE NULL::text +pg_stat_progress_cluster| SELECT pid, + datid, + datname, + relid, + CASE + WHEN (command = ANY (ARRAY['CLUSTER'::text, 'VACUUM FULL'::text])) THEN command + WHEN (repack_index_relid = (0)::oid) THEN 'VACUUM FULL'::text + ELSE 'CLUSTER'::text END AS command, - CASE s.param2 - WHEN 0 THEN 'initializing'::text - WHEN 1 THEN 'seq scanning heap'::text - WHEN 2 THEN 'index scanning heap'::text - WHEN 3 THEN 'sorting tuples'::text - WHEN 4 THEN 'writing new heap'::text - WHEN 5 THEN 'swapping relation files'::text - WHEN 6 THEN 'rebuilding index'::text - WHEN 7 THEN 'performing final cleanup'::text - ELSE NULL::text - END AS phase, - (s.param3)::oid AS cluster_index_relid, - s.param4 AS heap_tuples_scanned, - s.param5 AS heap_tuples_written, - s.param6 AS heap_blks_total, - s.param7 AS heap_blks_scanned, - s.param8 AS index_rebuild_count - FROM (pg_stat_get_progress_info('CLUSTER'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) - LEFT JOIN pg_database d ON ((s.datid = d.oid))); + phase, + repack_index_relid AS cluster_index_relid, + heap_tuples_scanned, + heap_tuples_written, + heap_blks_total, + heap_blks_scanned, + index_rebuild_count + FROM pg_stat_progress_repack; pg_stat_progress_copy| SELECT s.pid, s.datid, d.datname, @@ -2081,6 +2070,35 @@ pg_stat_progress_create_index| SELECT s.pid, s.param15 AS partitions_done FROM (pg_stat_get_progress_info('CREATE INDEX'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) LEFT JOIN pg_database d ON ((s.datid = d.oid))); +pg_stat_progress_repack| SELECT s.pid, + s.datid, + d.datname, + s.relid, + CASE s.param1 + WHEN 1 THEN 'CLUSTER'::text + WHEN 2 THEN 'REPACK'::text + WHEN 3 THEN 'VACUUM FULL'::text + ELSE NULL::text + END AS command, + CASE s.param2 + WHEN 0 THEN 'initializing'::text + WHEN 1 THEN 'seq scanning heap'::text + WHEN 2 THEN 'index scanning heap'::text + WHEN 3 THEN 'sorting tuples'::text + WHEN 4 THEN 'writing new heap'::text + WHEN 5 THEN 'swapping relation files'::text + WHEN 6 THEN 'rebuilding index'::text + WHEN 7 THEN 'performing final cleanup'::text + ELSE NULL::text + END AS phase, + (s.param3)::oid AS repack_index_relid, + s.param4 AS heap_tuples_scanned, + s.param5 AS heap_tuples_written, + s.param6 AS heap_blks_total, + s.param7 AS heap_blks_scanned, + s.param8 AS index_rebuild_count + FROM (pg_stat_get_progress_info('REPACK'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) + LEFT JOIN pg_database d ON ((s.datid = d.oid))); pg_stat_progress_vacuum| SELECT s.pid, s.datid, d.datname, diff --git a/src/test/regress/sql/cluster.sql b/src/test/regress/sql/cluster.sql index b7115f86104..c976823a3cb 100644 --- a/src/test/regress/sql/cluster.sql +++ b/src/test/regress/sql/cluster.sql @@ -76,7 +76,6 @@ INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass ORDER BY 1; - SELECT relname, relkind, EXISTS(SELECT 1 FROM pg_class WHERE oid = c.reltoastrelid) AS hastoast FROM pg_class c WHERE relname LIKE 'clstr_tst%' ORDER BY relname; @@ -229,6 +228,24 @@ SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM o CLUSTER clstrpart; ALTER TABLE clstrpart SET WITHOUT CLUSTER; ALTER TABLE clstrpart CLUSTER ON clstrpart_idx; + +-- Check that REPACK sets new relfilenodes: it should process exactly the same +-- tables as CLUSTER did. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart USING INDEX clstrpart_idx; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + +-- And finally the same for REPACK w/o index. +DROP TABLE old_cluster_info; +DROP TABLE new_cluster_info; +CREATE TEMP TABLE old_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +REPACK clstrpart; +CREATE TEMP TABLE new_cluster_info AS SELECT relname, level, relfilenode, relkind FROM pg_partition_tree('clstrpart'::regclass) AS tree JOIN pg_class c ON c.oid=tree.relid ; +SELECT relname, old.level, old.relkind, old.relfilenode = new.relfilenode FROM old_cluster_info AS old JOIN new_cluster_info AS new USING (relname) ORDER BY relname COLLATE "C"; + DROP TABLE clstrpart; -- Ownership of partitions is checked @@ -313,6 +330,57 @@ EXPLAIN (COSTS OFF) SELECT * FROM clstr_expression WHERE -a = -3 ORDER BY -a, b; SELECT * FROM clstr_expression WHERE -a = -3 ORDER BY -a, b; COMMIT; +---------------------------------------------------------------------- +-- +-- REPACK +-- +---------------------------------------------------------------------- + +-- REPACK handles individual tables identically to CLUSTER, but it's worth +-- checking if it handles table hierarchies identically as well. +REPACK clstr_tst USING INDEX clstr_tst_c; + +-- Verify that inheritance link still works +INSERT INTO clstr_tst_inh VALUES (0, 100, 'in child table 2'); +SELECT a,b,c,substring(d for 30), length(d) from clstr_tst; + +-- Verify that foreign key link still works +INSERT INTO clstr_tst (b, c) VALUES (1111, 'this should fail'); + +SELECT conname FROM pg_constraint WHERE conrelid = 'clstr_tst'::regclass +ORDER BY 1; + +-- Verify partial analyze works +REPACK (ANALYZE) clstr_tst (a); +REPACK (ANALYZE) clstr_tst; +REPACK (VERBOSE) clstr_tst (a); + +-- REPACK w/o argument performs no ordering, so we can only check which tables +-- have the relfilenode changed. +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_old AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); + +SET SESSION AUTHORIZATION regress_clstr_user; +SET client_min_messages = ERROR; -- order of "skipping" warnings may vary +REPACK; +RESET client_min_messages; + +RESET SESSION AUTHORIZATION; +CREATE TEMP TABLE relnodes_new AS +(SELECT relname, relfilenode +FROM pg_class +WHERE relname IN ('clstr_1', 'clstr_2', 'clstr_3')); + +-- Do the actual comparison. Unlike CLUSTER, clstr_3 should have been +-- processed because there is nothing like clustering index here. +SELECT o.relname FROM relnodes_old o +JOIN relnodes_new n ON o.relname = n.relname +WHERE o.relfilenode <> n.relfilenode +ORDER BY o.relname; + -- clean up DROP TABLE clustertest; DROP TABLE clstr_1; diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 018b5919cf6..0c7dd5d09e8 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2544,6 +2544,8 @@ ReorderBufferTupleCidKey ReorderBufferUpdateProgressTxnCB ReorderTuple RepOriginId +RepackCommand +RepackStmt ReparameterizeForeignPathByChild_function ReplaceVarsFromTargetList_context ReplaceVarsNoMatchOption -- 2.47.3 --ecl2euvoqutt27wj Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v25-0002-Refactor-index_concurrently_create_copy-for-use-.patch" ^ permalink raw reply [nested|flat] 11+ messages in thread
end of thread, other threads:[~2025-07-26 17:57 UTC | newest] Thread overview: 11+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-07-09 10:44 [PATCH v33 1/2] Unique key jesperpedersen <[email protected]> 2019-07-09 10:44 [PATCH] Unique key jesperpedersen <[email protected]> 2019-07-09 10:44 [PATCH v31 1/2] Unique key jesperpedersen <[email protected]> 2019-07-09 10:44 [PATCH v32 1/2] Unique key jesperpedersen <[email protected]> 2019-08-02 11:52 [PATCH 1/2] Unique key jesperpedersen <[email protected]> 2019-11-11 13:49 [PATCH 1/2] Unique key jesperpedersen <[email protected]> 2019-11-15 14:46 [PATCH 1/2] Unique key jesperpedersen <[email protected]> 2020-03-24 16:04 [PATCH v33 1/2] Unique key Dmitrii Dolgov <[email protected]> 2020-03-24 16:04 [PATCH v33 1/2] Unique key Dmitrii Dolgov <[email protected]> 2020-03-24 16:04 [PATCH v33 1/2] Unique key Dmitrii Dolgov <[email protected]> 2025-07-26 17:57 [PATCH v25 1/4] Add REPACK command Álvaro Herrera <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox