($INBOX_DIR/description missing)
help / color / mirror / Atom feed[PATCH v1 1/7] Row pattern recognition patch for raw parser.
81+ messages / 12 participants
[nested] [flat]
* [PATCH v1 1/7] Row pattern recognition patch for raw parser.
@ 2023-06-25 11:48 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-06-25 11:48 UTC (permalink / raw)
---
src/backend/parser/gram.y | 218 +++++++++++++++++++++++++++++---
src/include/nodes/parsenodes.h | 54 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 266 insertions(+), 15 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 39ab7eac0d..9520b5a07f 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -453,8 +455,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +557,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +640,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -645,7 +655,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> hash_partbound
%type <defelt> hash_partbound_elem
-
%type <node> json_format_clause_opt
json_value_expr
json_output_clause_opt
@@ -705,7 +714,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -721,7 +730,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -733,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -745,9 +754,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
- POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN PLACING PLANS POLICY
+ POSITION PRECEDING PRECISION PREMUTE PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
QUOTE
@@ -757,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -855,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15773,7 +15784,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15781,10 +15793,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15808,6 +15822,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -15967,6 +16006,139 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO FIRST_P ColId %prec FIRST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_FIRST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ * Shift/reduce
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17060,6 +17232,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17088,6 +17261,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17130,9 +17304,12 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
| PLANS
| POLICY
| PRECEDING
+ | PREMUTE
| PREPARE
| PREPARED
| PRESERVE
@@ -17180,6 +17357,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17205,6 +17383,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17389,6 +17568,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17551,6 +17731,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17626,6 +17807,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17673,6 +17855,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17726,11 +17909,14 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
| PLACING
| PLANS
| POLICY
| POSITION
| PRECEDING
+ | PREMUTE
| PREPARE
| PREPARED
| PRESERVE
@@ -17782,6 +17968,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17813,6 +18000,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index b3bec90e52..22e1a01a0e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -548,6 +548,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to> types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -563,6 +601,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1482,6 +1522,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1513,6 +1558,15 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern Recognition AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern Recognition DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern Recognition PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern Recognition PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f5b2e61ca5..12603b311c 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -263,6 +265,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -324,12 +327,15 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("preceding", PRECEDING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("precision", PRECISION, COL_NAME_KEYWORD, AS_LABEL)
+PG_KEYWORD("premute", PREMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("prepare", PREPARE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("prepared", PREPARED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("preserve", PRESERVE, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -383,6 +389,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -414,6 +421,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Sun_Jun_25_21_05_09_2023_126)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v1-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v2 1/7] Row pattern recognition patch for raw parser.
@ 2023-06-26 08:05 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-06-26 08:05 UTC (permalink / raw)
---
src/backend/parser/gram.y | 218 +++++++++++++++++++++++++++++---
src/include/nodes/parsenodes.h | 54 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 266 insertions(+), 15 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 39ab7eac0d..5cd3ebaa98 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -453,8 +455,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +557,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +640,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -645,7 +655,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> hash_partbound
%type <defelt> hash_partbound_elem
-
%type <node> json_format_clause_opt
json_value_expr
json_output_clause_opt
@@ -705,7 +714,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -721,7 +730,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -733,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -745,9 +754,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
- POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN PLACING PLANS POLICY
+ POSITION PRECEDING PRECISION PREMUTE PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
QUOTE
@@ -757,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -855,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15773,7 +15784,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15781,10 +15793,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15808,6 +15822,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -15967,6 +16006,139 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO FIRST_P ColId %prec FIRST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_FIRST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ * Shift/reduce
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17060,6 +17232,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17088,6 +17261,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17130,9 +17304,12 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
| PLANS
| POLICY
| PRECEDING
+ | PREMUTE
| PREPARE
| PREPARED
| PRESERVE
@@ -17180,6 +17357,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17205,6 +17383,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17389,6 +17568,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17551,6 +17731,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17626,6 +17807,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17673,6 +17855,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17726,11 +17909,14 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
| PLACING
| PLANS
| POLICY
| POSITION
| PRECEDING
+ | PREMUTE
| PREPARE
| PREPARED
| PRESERVE
@@ -17782,6 +17968,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17813,6 +18000,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index b3bec90e52..22e1a01a0e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -548,6 +548,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to> types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -563,6 +601,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1482,6 +1522,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1513,6 +1558,15 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern Recognition AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern Recognition DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern Recognition PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern Recognition PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f5b2e61ca5..12603b311c 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -263,6 +265,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -324,12 +327,15 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("preceding", PRECEDING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("precision", PRECISION, COL_NAME_KEYWORD, AS_LABEL)
+PG_KEYWORD("premute", PREMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("prepare", PREPARE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("prepared", PREPARED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("preserve", PRESERVE, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -383,6 +389,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -414,6 +421,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Mon_Jun_26_17_45_07_2023_724)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v2-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v3 1/7] Row pattern recognition patch for raw parser.
@ 2023-07-26 10:49 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-07-26 10:49 UTC (permalink / raw)
---
src/backend/parser/gram.y | 218 +++++++++++++++++++++++++++++---
src/include/nodes/parsenodes.h | 54 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 266 insertions(+), 15 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 856d5dee0e..a4c97c28ff 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -453,8 +455,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +557,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +640,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +669,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +711,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +727,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +740,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,9 +752,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
- POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN PLACING PLANS POLICY
+ POSITION PRECEDING PRECISION PREMUTE PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
QUOTE
@@ -755,12 +764,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +863,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15818,7 +15829,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15826,10 +15838,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15853,6 +15867,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16012,6 +16051,139 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO FIRST_P ColId %prec FIRST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_FIRST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ * Shift/reduce
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17107,6 +17279,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17134,6 +17307,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17176,9 +17350,12 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
| PLANS
| POLICY
| PRECEDING
+ | PREMUTE
| PREPARE
| PREPARED
| PRESERVE
@@ -17226,6 +17403,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17251,6 +17429,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17438,6 +17617,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17600,6 +17780,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17675,6 +17856,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17724,6 +17906,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17777,11 +17960,14 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
| PLACING
| PLANS
| POLICY
| POSITION
| PRECEDING
+ | PREMUTE
| PREPARE
| PREPARED
| PRESERVE
@@ -17833,6 +18019,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17864,6 +18051,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 228cdca0f1..c56c0d71c2 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,15 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern Recognition AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern Recognition DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern Recognition PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern Recognition PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..c799770025 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,12 +329,15 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("preceding", PRECEDING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("precision", PRECISION, COL_NAME_KEYWORD, AS_LABEL)
+PG_KEYWORD("premute", PREMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("prepare", PREPARE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("prepared", PREPARED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("preserve", PRESERVE, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Wed_Jul_26_21_21_34_2023_317)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v3-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v4 1/7] Row pattern recognition patch for raw parser.
@ 2023-08-09 07:56 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-08-09 07:56 UTC (permalink / raw)
---
src/backend/parser/gram.y | 216 +++++++++++++++++++++++++++++---
src/include/nodes/parsenodes.h | 56 +++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 267 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 15ece871a0..62c1919538 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -453,8 +455,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +557,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +640,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +669,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +711,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +727,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +740,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +752,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +764,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +863,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15832,7 +15843,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15840,10 +15852,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15867,6 +15881,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16026,6 +16065,139 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO FIRST_P ColId %prec FIRST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_FIRST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ * Shift/reduce
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17121,6 +17293,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17148,6 +17321,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17190,6 +17364,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17240,6 +17417,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17265,6 +17443,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17452,6 +17631,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17614,6 +17794,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17689,6 +17870,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17738,6 +17920,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17791,6 +17974,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17847,6 +18033,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17878,6 +18065,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fe003ded50..c78bd849f9 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..431e0625f1 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Wed_Aug__9_17_41_12_2023_134)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v4-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v5 1/7] Row pattern recognition patch for raw parser.
@ 2023-09-02 06:32 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-09-02 06:32 UTC (permalink / raw)
---
src/backend/parser/gram.y | 216 +++++++++++++++++++++++++++++---
src/include/nodes/parsenodes.h | 56 +++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 267 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..70409cdc9a 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -453,8 +455,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +557,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +640,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +669,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +711,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +727,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +740,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +752,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +764,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +863,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15857,7 +15868,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15865,10 +15877,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15892,6 +15906,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16051,6 +16090,139 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO FIRST_P ColId %prec FIRST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_FIRST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ * Shift/reduce
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17146,6 +17318,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17173,6 +17346,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17215,6 +17389,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17265,6 +17442,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17290,6 +17468,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17477,6 +17656,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17639,6 +17819,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17714,6 +17895,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17763,6 +17945,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17816,6 +17999,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17872,6 +18058,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17903,6 +18090,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fef4c714b8..4cc1fb417b 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Sat_Sep__2_15_52_35_2023_273)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v5-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v6 1/7] Row pattern recognition patch for raw parser.
@ 2023-09-12 05:22 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-09-12 05:22 UTC (permalink / raw)
---
src/backend/parser/gram.y | 216 +++++++++++++++++++++++++++++---
src/include/nodes/parsenodes.h | 56 +++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 267 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..70409cdc9a 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -453,8 +455,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +557,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +640,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +669,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +711,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +727,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +740,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +752,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +764,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +863,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15857,7 +15868,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15865,10 +15877,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15892,6 +15906,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16051,6 +16090,139 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO FIRST_P ColId %prec FIRST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_FIRST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ * Shift/reduce
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17146,6 +17318,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17173,6 +17346,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17215,6 +17389,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17265,6 +17442,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17290,6 +17468,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17477,6 +17656,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17639,6 +17819,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17714,6 +17895,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17763,6 +17945,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17816,6 +17999,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17872,6 +18058,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17903,6 +18090,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fef4c714b8..657651df1d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Tue_Sep_12_15_18_43_2023_359)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v6-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v7 1/7] Row pattern recognition patch for raw parser.
@ 2023-09-22 04:53 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-09-22 04:53 UTC (permalink / raw)
---
src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 56 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 273 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..74c2069050 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -278,6 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ RPSkipTo skipto;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -453,8 +456,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +558,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
+%type <skipto> first_or_last
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +642,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +671,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +713,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +754,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15857,7 +15870,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15865,10 +15879,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15892,6 +15908,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16051,6 +16092,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = ST_FIRST_VARIABLE; }
+ | LAST_P { $$ = ST_LAST_VARIABLE; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17146,6 +17324,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17173,6 +17352,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17215,6 +17395,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17265,6 +17448,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17290,6 +17474,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17477,6 +17662,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17639,6 +17825,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17714,6 +17901,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17763,6 +17951,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17816,6 +18005,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17872,6 +18064,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17903,6 +18096,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fef4c714b8..657651df1d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Fri_Sep_22_14_16_40_2023_530)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v7-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v8 1/7] Row pattern recognition patch for raw parser.
@ 2023-09-25 05:01 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-09-25 05:01 UTC (permalink / raw)
---
src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 56 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 273 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..74c2069050 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -278,6 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ RPSkipTo skipto;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -453,8 +456,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +558,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
+%type <skipto> first_or_last
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +642,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +671,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +713,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +754,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15857,7 +15870,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15865,10 +15879,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15892,6 +15908,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16051,6 +16092,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = ST_FIRST_VARIABLE; }
+ | LAST_P { $$ = ST_LAST_VARIABLE; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17146,6 +17324,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17173,6 +17352,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17215,6 +17395,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17265,6 +17448,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17290,6 +17474,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17477,6 +17662,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17639,6 +17825,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17714,6 +17901,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17763,6 +17951,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17816,6 +18005,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17872,6 +18064,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17903,6 +18096,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index fef4c714b8..657651df1d 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Mon_Sep_25_14_26_30_2023_752)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v8-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v9 1/7] Row pattern recognition patch for raw parser.
@ 2023-10-04 05:51 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-10-04 05:51 UTC (permalink / raw)
---
src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 56 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 273 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index e56cbe77cb..730c51bc87 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -278,6 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ RPSkipTo skipto;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -453,8 +456,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +558,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
+%type <skipto> first_or_last
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +642,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +671,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +713,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +754,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15892,7 +15905,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15900,10 +15914,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15927,6 +15943,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16086,6 +16127,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = ST_FIRST_VARIABLE; }
+ | LAST_P { $$ = ST_LAST_VARIABLE; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17181,6 +17359,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17208,6 +17387,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17250,6 +17430,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17300,6 +17483,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17325,6 +17509,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17512,6 +17697,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17674,6 +17860,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17749,6 +17936,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17798,6 +17986,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17851,6 +18040,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17907,6 +18099,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17938,6 +18131,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index f637937cd2..31b04d6919 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Wed_Oct__4_15_03_28_2023_821)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v9-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v10 1/7] Row pattern recognition patch for raw parser.
@ 2023-10-22 02:22 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-10-22 02:22 UTC (permalink / raw)
---
src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 56 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 273 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c224df4ecc..e09eb061f8 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -278,6 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ RPSkipTo skipto;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -453,8 +456,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +558,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
+%type <skipto> first_or_last
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +642,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +671,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +713,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +754,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15901,7 +15914,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15909,10 +15923,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15936,6 +15952,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16095,6 +16136,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = ST_FIRST_VARIABLE; }
+ | LAST_P { $$ = ST_LAST_VARIABLE; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17190,6 +17368,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17217,6 +17396,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17259,6 +17439,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17309,6 +17492,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17334,6 +17518,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17521,6 +17706,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17683,6 +17869,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17758,6 +17945,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17807,6 +17995,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17860,6 +18049,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17916,6 +18108,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17947,6 +18140,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index f637937cd2..31b04d6919 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -547,6 +547,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -562,6 +600,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1483,6 +1523,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1514,6 +1559,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Sun_Oct_22_11_39_20_2023_140)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v10-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v11 1/7] Row pattern recognition patch for raw parser.
@ 2023-11-08 06:57 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-11-08 06:57 UTC (permalink / raw)
---
src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 56 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 273 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c224df4ecc..e09eb061f8 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -278,6 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ RPSkipTo skipto;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -453,8 +456,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +558,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
+%type <skipto> first_or_last
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +642,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +671,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +713,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +754,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -853,6 +865,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15901,7 +15914,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15909,10 +15923,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15936,6 +15952,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16095,6 +16136,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = ST_FIRST_VARIABLE; }
+ | LAST_P { $$ = ST_LAST_VARIABLE; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17190,6 +17368,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17217,6 +17396,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17259,6 +17439,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17309,6 +17492,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17334,6 +17518,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17521,6 +17706,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17683,6 +17869,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17758,6 +17945,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17807,6 +17995,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17860,6 +18049,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17916,6 +18108,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17947,6 +18140,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index e494309da8..094c603887 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -540,6 +540,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -555,6 +593,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1476,6 +1516,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1507,6 +1552,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Wed_Nov__8_16_37_05_2023_872)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v11-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v12 1/7] Row pattern recognition patch for raw parser.
@ 2023-12-04 11:23 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2023-12-04 11:23 UTC (permalink / raw)
---
src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 56 ++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 273 insertions(+), 14 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index d631ac89a9..5a77fca17f 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -251,6 +251,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DefElem *defelt;
SortBy *sortby;
WindowDef *windef;
+ RPCommonSyntax *rpcom;
+ RPSubsetItem *rpsubset;
JoinExpr *jexpr;
IndexElem *ielem;
StatsElem *selem;
@@ -278,6 +280,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ RPSkipTo skipto;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -453,8 +456,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TriggerTransitions TriggerReferencing
vacuum_relation_list opt_vacuum_relation_list
drop_option_list pub_obj_list
-
-%type <node> opt_routine_body
+ row_pattern_measure_list row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list row_pattern_subset_rhs
+ row_pattern
+%type <rpsubset> row_pattern_subset_item
+%type <node> opt_routine_body row_pattern_term
%type <groupclause> group_clause
%type <list> group_by_list
%type <node> group_by_item empty_grouping_set rollup_clause cube_clause
@@ -551,6 +558,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <range> relation_expr_opt_alias
%type <node> tablesample_clause opt_repeatable_clause
%type <target> target_el set_target insert_column_item
+ row_pattern_measure_item row_pattern_definition
+%type <skipto> first_or_last
%type <str> generic_option_name
%type <node> generic_option_arg
@@ -633,6 +642,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <list> window_clause window_definition_list opt_partition_clause
%type <windef> window_definition over_clause window_specification
opt_frame_clause frame_extent frame_bound
+%type <rpcom> opt_row_pattern_common_syntax opt_row_pattern_skip_to
+%type <boolean> opt_row_pattern_initial_or_seek
+%type <list> opt_row_pattern_measures
%type <ival> opt_window_exclusion_clause
%type <str> opt_existing_window_name
%type <boolean> opt_if_not_exists
@@ -659,7 +671,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
* They must be listed first so that their numeric codes do not depend on
@@ -702,7 +713,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -718,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -731,7 +742,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -743,8 +754,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -755,12 +766,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -866,6 +878,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT
+%nonassoc MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15914,7 +15927,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15922,10 +15936,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = $7;
n->location = @1;
$$ = n;
}
@@ -15949,6 +15965,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16108,6 +16149,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $1->rpSkipTo;
+ n->rpSkipVariable = $1->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = ST_FIRST_VARIABLE; }
+ | LAST_P { $$ = ST_LAST_VARIABLE; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17217,6 +17395,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17244,6 +17423,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17286,6 +17466,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17336,6 +17519,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17361,6 +17545,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17548,6 +17733,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17710,6 +17896,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17785,6 +17972,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17834,6 +18022,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17887,6 +18076,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17943,6 +18135,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -17974,6 +18167,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index e494309da8..094c603887 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -540,6 +540,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -555,6 +593,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1476,6 +1516,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1507,6 +1552,17 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..2804333b53 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index f589112d5e..6640090910 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Fri_Dec__8_10_16_13_2023_489)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v12-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v13 1/8] Row pattern recognition patch for raw parser.
@ 2024-01-22 09:45 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-01-22 09:45 UTC (permalink / raw)
---
src/backend/parser/gram.y | 220 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 57 +++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 275 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 3460fea56b..84eb88ac2a 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -660,6 +660,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -703,7 +718,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -719,7 +734,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -732,7 +747,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -744,8 +759,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -756,12 +771,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -867,6 +883,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15928,7 +15945,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15936,10 +15954,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -15963,6 +15983,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16122,6 +16167,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17250,6 +17432,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17277,6 +17460,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17319,6 +17503,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17369,6 +17556,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17394,6 +17582,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17581,6 +17770,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17743,6 +17933,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17818,6 +18009,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17867,6 +18059,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17920,6 +18113,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17976,6 +18172,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18007,6 +18204,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index b3181f34ae..64e9df0a48 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -540,6 +540,44 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -555,6 +593,8 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1476,6 +1516,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1507,6 +1552,18 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable;/* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+ /* Row Pattern PATTERN regular expression quantifier ('+' or ''. list of String) */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 2331acac09..6b3734a865 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 99d6515736..1e43cef6f4 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Mon_Jan_22_19_26_18_2024_011)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v13-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v14 1/8] Row pattern recognition patch for raw parser.
@ 2024-02-28 13:59 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-02-28 13:59 UTC (permalink / raw)
---
src/backend/parser/gram.y | 220 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 285 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 130f7fc7c3..4581dd81aa 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -660,6 +660,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -703,7 +718,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -719,7 +734,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -732,7 +747,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -744,8 +759,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -756,12 +771,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -867,6 +883,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -15935,7 +15952,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -15943,10 +15961,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -15970,6 +15990,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16129,6 +16174,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17257,6 +17439,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17284,6 +17467,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17326,6 +17510,9 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17376,6 +17563,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17401,6 +17589,7 @@ unreserved_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17588,6 +17777,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17750,6 +17940,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -17825,6 +18016,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -17874,6 +18066,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINVALUE
@@ -17927,6 +18120,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -17983,6 +18179,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18014,6 +18211,7 @@ bare_label_keyword:
| STRICT_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index baa6a97c7e..14b9d328b7 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -540,6 +540,48 @@ typedef struct SortBy
int location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -555,6 +597,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1476,6 +1521,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1507,6 +1557,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 2331acac09..6b3734a865 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -128,6 +128,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -212,6 +213,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -265,6 +267,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD, AS_LABEL)
@@ -326,6 +329,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -385,6 +391,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -416,6 +423,7 @@ PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 99d6515736..1e43cef6f4 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Thu_Feb_29_09_19_54_2024_640)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v14-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v15 1/8] Row pattern recognition patch for raw parser.
@ 2024-03-28 10:30 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-03-28 10:30 UTC (permalink / raw)
---
src/backend/parser/gram.y | 220 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 285 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c1b0cff1c9..45817b296e 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -671,6 +671,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -714,7 +729,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -730,7 +745,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -743,7 +758,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
@@ -755,8 +770,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PERIOD PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATTERN_P PERIOD PERMUTE PLACING PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -767,12 +782,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -878,6 +894,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16037,7 +16054,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16045,10 +16063,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16072,6 +16092,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16231,6 +16276,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17441,6 +17623,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17469,6 +17652,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17512,7 +17696,10 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLANS
| POLICY
| PRECEDING
@@ -17564,6 +17751,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17590,6 +17778,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17782,6 +17971,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -17945,6 +18135,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18022,6 +18213,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18075,6 +18267,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18130,7 +18323,10 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLACING
| PLANS
| POLICY
@@ -18188,6 +18384,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18220,6 +18417,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index b89baef95d..28cf577089 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -549,6 +549,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -564,6 +606,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1516,6 +1561,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1547,6 +1597,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 57514d064b..4a47b13c34 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -272,6 +274,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -335,7 +338,10 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("period", PERIOD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -396,6 +402,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -428,6 +435,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 5b781d87a9..66935afff8 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Thu_Mar_28_19_59_25_2024_076)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v15-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v16 1/8] Row pattern recognition patch for raw parser.
@ 2024-04-12 06:49 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-04-12 06:49 UTC (permalink / raw)
---
src/backend/parser/gram.y | 221 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 285 insertions(+), 12 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 0523f7e891..8837fd4fa1 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -680,6 +680,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -723,7 +738,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -739,7 +754,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -752,7 +767,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
@@ -764,9 +779,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PATH
- PERIOD PLACING PLAN PLANS POLICY
-
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PAST
+ PATH PATTERN_P PERIOD PERMUTE PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -777,12 +791,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SPLIT SOURCE SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TARGET TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -891,6 +906,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED NESTED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT PATH
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16279,7 +16295,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16287,10 +16304,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16314,6 +16333,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16473,6 +16517,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17683,6 +17864,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17711,6 +17893,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17756,8 +17939,11 @@ unreserved_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLAN
| PLANS
| POLICY
@@ -17810,6 +17996,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17838,6 +18025,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -18032,6 +18220,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -18195,6 +18384,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18272,6 +18462,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18326,6 +18517,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18383,8 +18575,11 @@ bare_label_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLACING
| PLAN
| PLANS
@@ -18443,6 +18638,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18477,6 +18673,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index f763f790b1..aef5fbf458 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -550,6 +550,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -565,6 +607,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1533,6 +1578,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1564,6 +1614,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f9a4afd472..df49492629 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -273,6 +275,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -338,8 +341,11 @@ PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partitions", PARTITIONS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("period", PERIOD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -401,6 +407,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -435,6 +442,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 5b781d87a9..66935afff8 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Fri_Apr_12_16_09_08_2024_262)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v16-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v17 1/8] Row pattern recognition patch for raw parser.
@ 2024-04-28 11:00 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-04-28 11:00 UTC (permalink / raw)
---
src/backend/parser/gram.y | 221 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 285 insertions(+), 12 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index e8b619926e..c09a0f7e4f 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -680,6 +680,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -723,7 +738,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -739,7 +754,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -752,7 +767,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
@@ -764,9 +779,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PATH
- PERIOD PLACING PLAN PLANS POLICY
-
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PAST
+ PATH PATTERN_P PERIOD PERMUTE PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -777,12 +791,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SPLIT SOURCE SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TARGET TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -891,6 +906,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED NESTED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT PATH
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16333,7 +16349,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16341,10 +16358,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16368,6 +16387,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16527,6 +16571,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17737,6 +17918,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17765,6 +17947,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17810,8 +17993,11 @@ unreserved_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLAN
| PLANS
| POLICY
@@ -17864,6 +18050,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17892,6 +18079,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -18086,6 +18274,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -18249,6 +18438,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18326,6 +18516,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18380,6 +18571,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18437,8 +18629,11 @@ bare_label_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLACING
| PLAN
| PLANS
@@ -18497,6 +18692,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18531,6 +18727,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index af80a5d38e..7e7dcf3c77 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -550,6 +550,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -565,6 +607,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1533,6 +1578,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1564,6 +1614,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f9a4afd472..df49492629 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -273,6 +275,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -338,8 +341,11 @@ PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partitions", PARTITIONS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("period", PERIOD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -401,6 +407,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -435,6 +442,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 5b781d87a9..66935afff8 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Sun_Apr_28_20_28_26_2024_444)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v17-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v18 1/8] Row pattern recognition patch for raw parser.
@ 2024-05-11 07:11 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-05-11 07:11 UTC (permalink / raw)
---
src/backend/parser/gram.y | 221 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 285 insertions(+), 12 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index e8b619926e..c09a0f7e4f 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -680,6 +680,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -723,7 +738,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -739,7 +754,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -752,7 +767,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
@@ -764,9 +779,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PATH
- PERIOD PLACING PLAN PLANS POLICY
-
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PAST
+ PATH PATTERN_P PERIOD PERMUTE PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -777,12 +791,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SPLIT SOURCE SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TARGET TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -891,6 +906,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED NESTED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT PATH
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16333,7 +16349,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16341,10 +16358,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16368,6 +16387,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16527,6 +16571,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17737,6 +17918,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17765,6 +17947,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17810,8 +17993,11 @@ unreserved_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLAN
| PLANS
| POLICY
@@ -17864,6 +18050,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17892,6 +18079,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -18086,6 +18274,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -18249,6 +18438,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18326,6 +18516,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18380,6 +18571,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18437,8 +18629,11 @@ bare_label_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLACING
| PLAN
| PLANS
@@ -18497,6 +18692,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18531,6 +18727,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 3ca06fc3af..9e65e9132c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -550,6 +550,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -565,6 +607,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1533,6 +1578,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1562,6 +1612,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f9a4afd472..df49492629 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -273,6 +275,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -338,8 +341,11 @@ PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partitions", PARTITIONS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("period", PERIOD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -401,6 +407,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -435,6 +442,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 5b781d87a9..66935afff8 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Sat_May_11_16_23_07_2024_789)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v18-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v19 1/8] Row pattern recognition patch for raw parser.
@ 2024-05-14 23:26 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-05-14 23:26 UTC (permalink / raw)
---
src/backend/parser/gram.y | 221 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 285 insertions(+), 12 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 18a0a2dc2b..010b3359d1 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -680,6 +680,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -723,7 +738,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -739,7 +754,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -752,7 +767,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
@@ -764,9 +779,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PATH
- PERIOD PLACING PLAN PLANS POLICY
-
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PAST
+ PATH PATTERN_P PERIOD PERMUTE PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -777,12 +791,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SPLIT SOURCE SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TARGET TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -891,6 +906,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED NESTED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT PATH
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16316,7 +16332,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16324,10 +16341,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16351,6 +16370,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16510,6 +16554,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17720,6 +17901,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17748,6 +17930,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17793,8 +17976,11 @@ unreserved_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLAN
| PLANS
| POLICY
@@ -17847,6 +18033,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17875,6 +18062,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -18069,6 +18257,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -18232,6 +18421,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18309,6 +18499,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18363,6 +18554,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18420,8 +18612,11 @@ bare_label_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLACING
| PLAN
| PLANS
@@ -18480,6 +18675,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18514,6 +18710,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index dcfd080dd5..54427003b3 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -550,6 +550,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -565,6 +607,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1533,6 +1578,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1562,6 +1612,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f9a4afd472..df49492629 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -273,6 +275,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -338,8 +341,11 @@ PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partitions", PARTITIONS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("period", PERIOD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -401,6 +407,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -435,6 +442,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 5b781d87a9..66935afff8 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Wed_May_15_09_02_03_2024_008)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v19-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v20 1/8] Row pattern recognition patch for raw parser.
@ 2024-05-24 02:26 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-05-24 02:26 UTC (permalink / raw)
---
src/backend/parser/gram.y | 222 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 9 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 288 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 4d582950b7..ef6f25b2f8 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -679,6 +679,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -722,7 +737,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -738,7 +753,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -751,7 +766,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
@@ -763,8 +778,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PATH
- PLACING PLAN PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PARTITIONS PASSING PASSWORD PAST
+ PATH PATTERN_P PERIOD PERMUTE PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -775,12 +790,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SPLIT SOURCE SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TARGET TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -889,6 +905,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED NESTED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT PATH
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16287,7 +16304,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16295,10 +16313,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16322,6 +16342,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16481,6 +16526,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17691,6 +17873,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17719,6 +17902,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17764,7 +17948,11 @@ unreserved_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
+ | PERIOD
+ | PERMUTE
| PLAN
| PLANS
| POLICY
@@ -17817,6 +18005,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17845,6 +18034,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -18039,6 +18229,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -18202,6 +18393,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18279,6 +18471,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18333,6 +18526,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18390,7 +18584,11 @@ bare_label_keyword:
| PARTITIONS
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
+ | PERIOD
+ | PERMUTE
| PLACING
| PLAN
| PLANS
@@ -18449,6 +18647,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18483,6 +18682,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ddfed02db2..c8a2eed08f 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -550,6 +550,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -565,6 +607,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1533,6 +1578,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1562,6 +1612,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f7fe834cf4..df49492629 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -273,6 +275,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -338,7 +341,11 @@ PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partitions", PARTITIONS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("period", PERIOD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -400,6 +407,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -434,6 +442,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 5b781d87a9..66935afff8 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Fri_May_24_11_39_19_2024_763)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v20-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v21 1/8] Row pattern recognition patch for raw parser.
@ 2024-08-26 04:32 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-08-26 04:32 UTC (permalink / raw)
---
src/backend/parser/gram.y | 221 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 286 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 84cef57a70..a8130da934 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -676,6 +676,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -719,7 +734,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -735,7 +750,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -748,7 +763,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
@@ -760,8 +775,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PATH
- PLACING PLAN PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATH PATTERN_P PERMUTE PLACING PLAN PLANS POLICY
+
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -772,12 +788,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SOURCE SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TARGET TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -886,6 +903,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED NESTED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT PATH
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16217,7 +16235,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16225,10 +16244,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16252,6 +16273,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16411,6 +16457,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17621,6 +17804,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17649,6 +17833,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17693,7 +17878,10 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
+ | PERMUTE
| PLAN
| PLANS
| POLICY
@@ -17745,6 +17933,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17772,6 +17961,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -17966,6 +18156,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -18129,6 +18320,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18206,6 +18398,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18260,6 +18453,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18316,7 +18510,10 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
+ | PERMUTE
| PLACING
| PLAN
| PLANS
@@ -18374,6 +18571,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18407,6 +18605,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 124d853e49..ea4901ed1e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -550,6 +550,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -565,6 +607,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1520,6 +1565,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1549,6 +1599,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f8659078ce..f1af381718 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -273,6 +275,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -337,7 +340,10 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -398,6 +404,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -431,6 +438,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 5b781d87a9..66935afff8 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Mon_Aug_26_13_39_47_2024_878)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v21-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* [PATCH v22 1/8] Row pattern recognition patch for raw parser.
@ 2024-09-19 04:48 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tatsuo Ishii @ 2024-09-19 04:48 UTC (permalink / raw)
---
src/backend/parser/gram.y | 221 ++++++++++++++++++++++++++++++--
src/include/nodes/parsenodes.h | 67 ++++++++++
src/include/parser/kwlist.h | 8 ++
src/include/parser/parse_node.h | 1 +
4 files changed, 286 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ab304ca989..7f111255c6 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -677,6 +677,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <target> row_pattern_measure_item
+ row_pattern_definition
+%type <node> opt_row_pattern_common_syntax
+ opt_row_pattern_skip_to
+ row_pattern_subset_item
+ row_pattern_term
+%type <list> opt_row_pattern_measures
+ row_pattern_measure_list
+ row_pattern_definition_list
+ opt_row_pattern_subset_clause
+ row_pattern_subset_list
+ row_pattern_subset_rhs
+ row_pattern
+%type <boolean> opt_row_pattern_initial_or_seek
+ first_or_last
/*
* Non-keyword token types. These are hard-wired into the "flex" lexer.
@@ -720,7 +735,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE
DATA_P DATABASE DAY_P DEALLOCATE DEC DECIMAL_P DECLARE DEFAULT DEFAULTS
- DEFERRABLE DEFERRED DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
+ DEFERRABLE DEFERRED DEFINE DEFINER DELETE_P DELIMITER DELIMITERS DEPENDS DEPTH DESC
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
@@ -736,7 +751,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
HANDLER HAVING HEADER_P HOLD HOUR_P
IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE
- INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P
+ INCLUDING INCREMENT INDENT INDEX INDEXES INHERIT INHERITS INITIAL INITIALLY INLINE_P
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
@@ -749,7 +764,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
- MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE MERGE_ACTION METHOD
+ MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MEASURES MERGE MERGE_ACTION METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
@@ -761,8 +776,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PATH
- PERIOD PLACING PLAN PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PAST
+ PATH PATTERN_P PERIOD PERMUTE PLACING PLAN PLANS POLICY
+
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -773,12 +789,13 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
RESET RESTART RESTRICT RETURN RETURNING RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
ROUTINE ROUTINES ROW ROWS RULE
- SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT
+ SAVEPOINT SCALAR SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SEEK SELECT
SEQUENCE SEQUENCES
+
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SOURCE SQL_P STABLE STANDALONE_P
START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
- SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
+ SUBSCRIPTION SUBSET SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TARGET TEMP TEMPLATE TEMPORARY TEXT_P THEN
TIES TIME TIMESTAMP TO TRAILING TRANSACTION TRANSFORM
@@ -887,6 +904,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%nonassoc UNBOUNDED NESTED /* ideally would have same precedence as IDENT */
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
SET KEYS OBJECT_P SCALAR VALUE_P WITH WITHOUT PATH
+ MEASURES AFTER INITIAL SEEK PATTERN_P
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
%left '*' '/' '%'
@@ -16252,7 +16270,8 @@ over_clause: OVER window_specification
;
window_specification: '(' opt_existing_window_name opt_partition_clause
- opt_sort_clause opt_frame_clause ')'
+ opt_sort_clause opt_row_pattern_measures opt_frame_clause
+ opt_row_pattern_common_syntax ')'
{
WindowDef *n = makeNode(WindowDef);
@@ -16260,10 +16279,12 @@ window_specification: '(' opt_existing_window_name opt_partition_clause
n->refname = $2;
n->partitionClause = $3;
n->orderClause = $4;
+ n->rowPatternMeasures = $5;
/* copy relevant fields of opt_frame_clause */
- n->frameOptions = $5->frameOptions;
- n->startOffset = $5->startOffset;
- n->endOffset = $5->endOffset;
+ n->frameOptions = $6->frameOptions;
+ n->startOffset = $6->startOffset;
+ n->endOffset = $6->endOffset;
+ n->rpCommonSyntax = (RPCommonSyntax *)$7;
n->location = @1;
$$ = n;
}
@@ -16287,6 +16308,31 @@ opt_partition_clause: PARTITION BY expr_list { $$ = $3; }
| /*EMPTY*/ { $$ = NIL; }
;
+/*
+ * ROW PATTERN_P MEASURES
+ */
+opt_row_pattern_measures: MEASURES row_pattern_measure_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_measure_list:
+ row_pattern_measure_item
+ { $$ = list_make1($1); }
+ | row_pattern_measure_list ',' row_pattern_measure_item
+ { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_measure_item:
+ a_expr AS ColLabel
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $3;
+ $$->indirection = NIL;
+ $$->val = (Node *) $1;
+ $$->location = @1;
+ }
+ ;
+
/*
* For frame clauses, we return a WindowDef, but only some fields are used:
* frameOptions, startOffset, and endOffset.
@@ -16446,6 +16492,143 @@ opt_window_exclusion_clause:
| /*EMPTY*/ { $$ = 0; }
;
+opt_row_pattern_common_syntax:
+opt_row_pattern_skip_to opt_row_pattern_initial_or_seek
+ PATTERN_P '(' row_pattern ')'
+ opt_row_pattern_subset_clause
+ DEFINE row_pattern_definition_list
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ((RPCommonSyntax *)$1)->rpSkipTo;
+ n->rpSkipVariable = ((RPCommonSyntax *)$1)->rpSkipVariable;
+ n->initial = $2;
+ n->rpPatterns = $5;
+ n->rpSubsetClause = $7;
+ n->rpDefs = $9;
+ $$ = (Node *) n;
+ }
+ | /*EMPTY*/ { $$ = NULL; }
+ ;
+
+opt_row_pattern_skip_to:
+ AFTER MATCH SKIP TO NEXT ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_NEXT_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP PAST LAST_P ROW
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ | AFTER MATCH SKIP TO first_or_last ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = $5? ST_FIRST_VARIABLE : ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = (Node *) n;
+ }
+/*
+ | AFTER MATCH SKIP TO LAST_P ColId %prec LAST_P
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_LAST_VARIABLE;
+ n->rpSkipVariable = $6;
+ $$ = n;
+ }
+ | AFTER MATCH SKIP TO ColId
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ n->rpSkipTo = ST_VARIABLE;
+ n->rpSkipVariable = $5;
+ $$ = n;
+ }
+*/
+ | /*EMPTY*/
+ {
+ RPCommonSyntax *n = makeNode(RPCommonSyntax);
+ /* temporary set default to ST_NEXT_ROW */
+ n->rpSkipTo = ST_PAST_LAST_ROW;
+ n->rpSkipVariable = NULL;
+ $$ = (Node *) n;
+ }
+ ;
+
+first_or_last:
+ FIRST_P { $$ = true; }
+ | LAST_P { $$ = false; }
+ ;
+
+opt_row_pattern_initial_or_seek:
+ INITIAL { $$ = true; }
+ | SEEK
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SEEK is not supported"),
+ errhint("Use INITIAL."),
+ parser_errposition(@1)));
+ }
+ | /*EMPTY*/ { $$ = true; }
+ ;
+
+row_pattern:
+ row_pattern_term { $$ = list_make1($1); }
+ | row_pattern row_pattern_term { $$ = lappend($1, $2); }
+ ;
+
+row_pattern_term:
+ ColId { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "", (Node *)makeString($1), NULL, @1); }
+ | ColId '*' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "*", (Node *)makeString($1), NULL, @1); }
+ | ColId '+' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "+", (Node *)makeString($1), NULL, @1); }
+ | ColId '?' { $$ = (Node *) makeSimpleA_Expr(AEXPR_OP, "?", (Node *)makeString($1), NULL, @1); }
+ ;
+
+opt_row_pattern_subset_clause:
+ SUBSET row_pattern_subset_list { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_list:
+ row_pattern_subset_item { $$ = list_make1($1); }
+ | row_pattern_subset_list ',' row_pattern_subset_item { $$ = lappend($1, $3); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_subset_item: ColId '=' '(' row_pattern_subset_rhs ')'
+ {
+ RPSubsetItem *n = makeNode(RPSubsetItem);
+ n->name = $1;
+ n->rhsVariable = $4;
+ $$ = (Node *) n;
+ }
+ ;
+
+row_pattern_subset_rhs:
+ ColId { $$ = list_make1(makeStringConst($1, @1)); }
+ | row_pattern_subset_rhs ',' ColId { $$ = lappend($1, makeStringConst($3, @1)); }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+row_pattern_definition_list:
+ row_pattern_definition { $$ = list_make1($1); }
+ | row_pattern_definition_list ',' row_pattern_definition { $$ = lappend($1, $3); }
+ ;
+
+row_pattern_definition:
+ ColId AS a_expr
+ {
+ $$ = makeNode(ResTarget);
+ $$->name = $1;
+ $$->indirection = NIL;
+ $$->val = (Node *) $3;
+ $$->location = @1;
+ }
+ ;
/*
* Supporting nonterminals for expressions.
@@ -17656,6 +17839,7 @@ unreserved_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INLINE_P
| INPUT_P
| INSENSITIVE
@@ -17684,6 +17868,7 @@ unreserved_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| METHOD
| MINUTE_P
@@ -17728,8 +17913,11 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLAN
| PLANS
| POLICY
@@ -17781,6 +17969,7 @@ unreserved_keyword:
| SEARCH
| SECOND_P
| SECURITY
+ | SEEK
| SEQUENCE
| SEQUENCES
| SERIALIZABLE
@@ -17808,6 +17997,7 @@ unreserved_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUPPORT
| SYSID
| SYSTEM_P
@@ -18002,6 +18192,7 @@ reserved_keyword:
| CURRENT_USER
| DEFAULT
| DEFERRABLE
+ | DEFINE
| DESC
| DISTINCT
| DO
@@ -18165,6 +18356,7 @@ bare_label_keyword:
| DEFAULTS
| DEFERRABLE
| DEFERRED
+ | DEFINE
| DEFINER
| DELETE_P
| DELIMITER
@@ -18242,6 +18434,7 @@ bare_label_keyword:
| INDEXES
| INHERIT
| INHERITS
+ | INITIAL
| INITIALLY
| INLINE_P
| INNER_P
@@ -18296,6 +18489,7 @@ bare_label_keyword:
| MATCHED
| MATERIALIZED
| MAXVALUE
+ | MEASURES
| MERGE
| MERGE_ACTION
| METHOD
@@ -18352,8 +18546,11 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PAST
| PATH
+ | PATTERN_P
| PERIOD
+ | PERMUTE
| PLACING
| PLAN
| PLANS
@@ -18411,6 +18608,7 @@ bare_label_keyword:
| SCROLL
| SEARCH
| SECURITY
+ | SEEK
| SELECT
| SEQUENCE
| SEQUENCES
@@ -18444,6 +18642,7 @@ bare_label_keyword:
| STRING_P
| STRIP_P
| SUBSCRIPTION
+ | SUBSET
| SUBSTRING
| SUPPORT
| SYMMETRIC
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index e62ce1b753..8e4f27a00f 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -552,6 +552,48 @@ typedef struct SortBy
ParseLoc location; /* operator location, or -1 if none/unknown */
} SortBy;
+/*
+ * AFTER MATCH row pattern skip to types in row pattern common syntax
+ */
+typedef enum RPSkipTo
+{
+ ST_NONE, /* AFTER MATCH omitted */
+ ST_NEXT_ROW, /* SKIP TO NEXT ROW */
+ ST_PAST_LAST_ROW, /* SKIP TO PAST LAST ROW */
+ ST_FIRST_VARIABLE, /* SKIP TO FIRST variable name */
+ ST_LAST_VARIABLE, /* SKIP TO LAST variable name */
+ ST_VARIABLE /* SKIP TO variable name */
+} RPSkipTo;
+
+/*
+ * Row Pattern SUBSET clause item
+ */
+typedef struct RPSubsetItem
+{
+ NodeTag type;
+ char *name; /* Row Pattern SUBSET clause variable name */
+ List *rhsVariable; /* Row Pattern SUBSET rhs variables (list of
+ * char *string) */
+} RPSubsetItem;
+
+/*
+ * RowPatternCommonSyntax - raw representation of row pattern common syntax
+ *
+ */
+typedef struct RPCommonSyntax
+{
+ NodeTag type;
+ RPSkipTo rpSkipTo; /* Row Pattern AFTER MATCH SKIP type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable name, if any */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ List *rpPatterns; /* PATTERN variables (list of A_Expr) */
+ List *rpSubsetClause; /* row pattern subset clause (list of
+ * RPSubsetItem), if any */
+ List *rpDefs; /* row pattern definitions clause (list of
+ * ResTarget) */
+} RPCommonSyntax;
+
/*
* WindowDef - raw representation of WINDOW and OVER clauses
*
@@ -567,6 +609,9 @@ typedef struct WindowDef
char *refname; /* referenced window name, if any */
List *partitionClause; /* PARTITION BY expression list */
List *orderClause; /* ORDER BY (list of SortBy) */
+ List *rowPatternMeasures; /* row pattern measures (list of
+ * ResTarget) */
+ RPCommonSyntax *rpCommonSyntax; /* row pattern common syntax */
int frameOptions; /* frame_clause options, see below */
Node *startOffset; /* expression for starting bound, if any */
Node *endOffset; /* expression for ending bound, if any */
@@ -1529,6 +1574,11 @@ typedef struct GroupingSet
* the orderClause might or might not be copied (see copiedOrder); the framing
* options are never copied, per spec.
*
+ * "defineClause" is Row Pattern Recognition DEFINE clause (list of
+ * TargetEntry). TargetEntry.resname represents row pattern definition
+ * variable name. "patternVariable" and "patternRegexp" represents PATTERN
+ * clause.
+ *
* The information relevant for the query jumbling is the partition clause
* type and its bounds.
*/
@@ -1558,6 +1608,23 @@ typedef struct WindowClause
Index winref; /* ID referenced by window functions */
/* did we copy orderClause from refname? */
bool copiedOrder pg_node_attr(query_jumble_ignore);
+ /* Row Pattern AFTER MACH SKIP clause */
+ RPSkipTo rpSkipTo; /* Row Pattern Skip To type */
+ char *rpSkipVariable; /* Row Pattern Skip To variable */
+ bool initial; /* true if <row pattern initial or seek> is
+ * initial */
+ /* Row Pattern DEFINE clause (list of TargetEntry) */
+ List *defineClause;
+ /* Row Pattern DEFINE variable initial names (list of String) */
+ List *defineInitial;
+ /* Row Pattern PATTERN variable name (list of String) */
+ List *patternVariable;
+
+ /*
+ * Row Pattern PATTERN regular expression quantifier ('+' or ''. list of
+ * String)
+ */
+ List *patternRegexp;
} WindowClause;
/*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 899d64ad55..f52d797615 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -129,6 +129,7 @@ PG_KEYWORD("default", DEFAULT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("defaults", DEFAULTS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferrable", DEFERRABLE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("deferred", DEFERRED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("define", DEFINE, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("definer", DEFINER, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delete", DELETE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("delimiter", DELIMITER, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -215,6 +216,7 @@ PG_KEYWORD("index", INDEX, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("indexes", INDEXES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherit", INHERIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inherits", INHERITS, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("initial", INITIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("initially", INITIALLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inline", INLINE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("inner", INNER_P, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
@@ -273,6 +275,7 @@ PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("matched", MATCHED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("measures", MEASURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge", MERGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("merge_action", MERGE_ACTION, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -337,8 +340,11 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("past", PAST, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("pattern", PATTERN_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("period", PERIOD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("permute", PERMUTE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -399,6 +405,7 @@ PG_KEYWORD("scroll", SCROLL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("search", SEARCH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("second", SECOND_P, UNRESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("security", SECURITY, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("seek", SEEK, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("select", SELECT, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequence", SEQUENCE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("sequences", SEQUENCES, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -432,6 +439,7 @@ PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("subset", SUBSET, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("support", SUPPORT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("symmetric", SYMMETRIC, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index 543df56814..2d270f443b 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -51,6 +51,7 @@ typedef enum ParseExprKind
EXPR_KIND_WINDOW_FRAME_RANGE, /* window frame clause with RANGE */
EXPR_KIND_WINDOW_FRAME_ROWS, /* window frame clause with ROWS */
EXPR_KIND_WINDOW_FRAME_GROUPS, /* window frame clause with GROUPS */
+ EXPR_KIND_RPR_DEFINE, /* DEFINE */
EXPR_KIND_SELECT_TARGET, /* SELECT target list item */
EXPR_KIND_INSERT_TARGET, /* INSERT target list item */
EXPR_KIND_UPDATE_SOURCE, /* UPDATE assignment source item */
--
2.25.1
----Next_Part(Thu_Sep_19_13_59_47_2024_608)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v22-0002-Row-pattern-recognition-patch-parse-analysis.patch"
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
@ 2024-11-26 22:11 Corey Huinker <[email protected]>
2024-11-27 05:08 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 2 replies; 81+ messages in thread
From: Corey Huinker @ 2024-11-26 22:11 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
> Comments on 0003:
>
> * If we commit 0003, is it a useful feature by itself or does it
> require that we commit some or all of 0004-0014? Which of these need to
> be in v18?
>
Useful by itself.
0004 seems needed to me, unless we're fine with ~50% bloat in pg_class on a
new-upgraded system, or we think inplace update are on their way out.
0005 is basically theoretical, it is only needed if we change the default
relpages on partitioned tables.
0006-0011 are the vacuumdb things being debated now. I've reached out to
some of the people I spoke to at PgConf.eu to get them to chime in.
0012 is now moot as a similar patch was committed Friday.
0013 is a cleanup/optimization.
0014 is the --no-data flag, which has no meaning without 0004, but 0004 in
no way requires it.
>
> * Why does binary upgrade cause statistics to be dumped? Can you just
> make pg_upgrade specify the appropriate set of flags?
>
That decision goes back a ways, I tried to dig in the archives last night
but I was getting a Server Error on postgresql.org.
Today I'm coming up with
https://www.postgresql.org/message-id/267624.1711756062%40sss.pgh.pa.us
which is actually more about whether stats import should be the default
(consensus: yesyesyes), and the binary_upgrade test may have been because
binary_upgrade shuts off data section stuff, but table stats are in the
data section. Happy to review the decision.
> * It looks like appendNamedArgument() is never called with
> positional=true?
>
That is currently the case. Currently all params are called with name/value
pairs, but in the past we had leading positionals followed by the stat-y
parameters in name-value pairs. I'll be refactoring it to remove the
positonal=T/F argument, which leaves just a list of name-type pairs, and
thus probably reduces the urge to make it an array of structs.
> * It's pretty awkward to use an array of arrays of strings when an
> array of structs might make more sense.
That pattern was introduced here:
https://www.postgresql.org/message-id/4afa70edab849ff16238d1100b6652404e9a4d9d.camel%40j-davis.com
:)
I'll be rebasing (that's done) and refactoring 0003 to get rid of the
positional column, and moving 0014 next to 0003 because they touch the same
files.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2024-11-27 05:08 ` Corey Huinker <[email protected]>
2024-12-07 00:44 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2024-11-27 05:08 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
>
> I'll be rebasing (that's done) and refactoring 0003 to get rid of the
> positional column, and moving 0014 next to 0003 because they touch the same
> files.
>
As promised, rebased (as of 8fcd80258bcf43dab93d877a5de0ce3f4d2bd471)
Things have been reordered here in a mostly-priority order:
0001 - Enable dumping stats in pg_dump/pg_upgrade. Now with less dead code!
0002 - The dump --no-data option. Very handy for diagnosing customer query
plans, but not in the critical path for v18. Still, so useful that we ought
to include it.
0003 - Re-enabling in-place updates because catalog bloat bad.
0004 - Combine two syscache lookups into one. Not strictly necessary, but
the second lookup is redundant if the first one grabs what we need.
0005-0010 vacuumdb changes (still up for debate) in baby steps.
0011 - The fix for clearing relation stats if the default for relpages
becomes -1 on inherited tables (which it hasn't, yet, and might never).
Attachments:
[text/x-patch] v33-0002-Add-no-data-option.patch (5.4K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/3-v33-0002-Add-no-data-option.patch)
download | inline diff:
From 0f616d4d44f7e29517ac348ded09da1fef1bcbd9 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 14 Nov 2024 04:58:17 -0500
Subject: [PATCH v33 02/11] Add --no-data option.
This option is useful for situations where someone wishes to test
query plans from a production database without copying production data.
---
src/bin/pg_dump/pg_backup.h | 2 ++
src/bin/pg_dump/pg_backup_archiver.c | 2 ++
src/bin/pg_dump/pg_dump.c | 7 ++++++-
src/bin/pg_dump/pg_restore.c | 5 ++++-
4 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 8fbb39d399..241855d017 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,6 +110,7 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
@@ -185,6 +186,7 @@ typedef struct _dumpOptions
int no_publications;
int no_subscriptions;
int no_statistics;
+ int no_data;
int no_toast_compression;
int no_unlogged_table_data;
int serializable_deferrable;
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index af8bb6bd12..b045c7aa7d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -186,6 +186,8 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 40f75130b2..161eb80859 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -494,6 +494,7 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
{"no-statistics", no_argument, &dopt.no_statistics, 1},
@@ -796,6 +797,9 @@ main(int argc, char **argv)
if (data_only && statistics_only)
pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+
if (statistics_only && dopt.no_statistics)
pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
@@ -812,7 +816,7 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpData = data_only || (!schema_only && !statistics_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
if (statistics_only)
@@ -1247,6 +1251,7 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
printf(_(" --no-statistics do not dump statistics\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 355f0439da..31c3cd32de 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -71,6 +71,7 @@ main(int argc, char **argv)
static int outputNoTableAm = 0;
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
+ static int no_data = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
@@ -132,6 +133,7 @@ main(int argc, char **argv)
{"no-subscriptions", no_argument, &no_subscriptions, 1},
{"filter", required_argument, NULL, 4},
{"no-statistics", no_argument, &no_statistics, 1},
+ {"no-data", no_argument, &no_data, 1},
{NULL, 0, NULL, 0}
};
@@ -375,7 +377,7 @@ main(int argc, char **argv)
/* set derivative flags */
opts->dumpSchema = (!data_only && !statistics_only);
- opts->dumpData = (!schema_only && !statistics_only);
+ opts->dumpData = (!no_data && !schema_only && !statistics_only);
opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
@@ -389,6 +391,7 @@ main(int argc, char **argv)
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
--
2.47.0
[text/x-patch] v33-0003-Enable-in-place-updates-for-pg_restore_relation_.patch (5.4K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/4-v33-0003-Enable-in-place-updates-for-pg_restore_relation_.patch)
download | inline diff:
From 4538ce000c39c53ec23bfbe9c903f5e47d28f573 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Mon, 4 Nov 2024 14:02:57 -0500
Subject: [PATCH v33 03/11] Enable in-place updates for
pg_restore_relation_stats.
This matches the behavior of the ANALYZE command, and would avoid
bloating pg_class in an upgrade situation wherein
pg_restore_relation_stats would be called for nearly every relation in
the database.
---
src/backend/statistics/relation_stats.c | 72 +++++++++++++++++++------
1 file changed, 55 insertions(+), 17 deletions(-)
diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c
index e619d5cf5b..24f646048c 100644
--- a/src/backend/statistics/relation_stats.c
+++ b/src/backend/statistics/relation_stats.c
@@ -22,6 +22,7 @@
#include "statistics/stat_utils.h"
#include "utils/fmgrprotos.h"
#include "utils/syscache.h"
+#include "utils/fmgroids.h"
#define DEFAULT_RELPAGES Int32GetDatum(0)
#define DEFAULT_RELTUPLES Float4GetDatum(-1.0)
@@ -50,13 +51,14 @@ static struct StatsArgInfo relarginfo[] =
[NUM_RELATION_STATS_ARGS] = {0}
};
-static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel);
+static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel,
+ bool inplace);
/*
* Internal function for modifying statistics for a relation.
*/
static bool
-relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
+relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace)
{
Oid reloid;
Relation crel;
@@ -68,6 +70,7 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
int ncols = 0;
TupleDesc tupdesc;
bool result = true;
+ void *inplace_state;
stats_check_required_arg(fcinfo, relarginfo, RELATION_ARG);
reloid = PG_GETARG_OID(RELATION_ARG);
@@ -87,7 +90,20 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
crel = table_open(RelationRelationId, RowExclusiveLock);
tupdesc = RelationGetDescr(crel);
- ctup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid));
+ if (inplace)
+ {
+ ScanKeyData key[1];
+
+ ctup = NULL;
+
+ ScanKeyInit(&key[0], Anum_pg_class_oid, BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(reloid));
+ systable_inplace_update_begin(crel, ClassOidIndexId, true, NULL, 1, key,
+ &ctup, &inplace_state);
+ }
+ else
+ ctup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid));
+
if (!HeapTupleIsValid(ctup))
{
ereport(elevel,
@@ -118,8 +134,13 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
}
else if (relpages != pgcform->relpages)
{
- replaces[ncols] = Anum_pg_class_relpages;
- values[ncols] = Int32GetDatum(relpages);
+ if (inplace)
+ pgcform->relpages = relpages;
+ else
+ {
+ replaces[ncols] = Anum_pg_class_relpages;
+ values[ncols] = Int32GetDatum(relpages);
+ }
ncols++;
}
}
@@ -137,8 +158,13 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
}
else if (reltuples != pgcform->reltuples)
{
- replaces[ncols] = Anum_pg_class_reltuples;
- values[ncols] = Float4GetDatum(reltuples);
+ if (inplace)
+ pgcform->reltuples = reltuples;
+ else
+ {
+ replaces[ncols] = Anum_pg_class_reltuples;
+ values[ncols] = Float4GetDatum(reltuples);
+ }
ncols++;
}
@@ -157,8 +183,13 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
}
else if (relallvisible != pgcform->relallvisible)
{
- replaces[ncols] = Anum_pg_class_relallvisible;
- values[ncols] = Int32GetDatum(relallvisible);
+ if (inplace)
+ pgcform->relallvisible = relallvisible;
+ else
+ {
+ replaces[ncols] = Anum_pg_class_relallvisible;
+ values[ncols] = Int32GetDatum(relallvisible);
+ }
ncols++;
}
}
@@ -166,13 +197,20 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
/* only update pg_class if there is a meaningful change */
if (ncols > 0)
{
- HeapTuple newtup;
+ if (inplace)
+ systable_inplace_update_finish(inplace_state, ctup);
+ else
+ {
+ HeapTuple newtup;
- newtup = heap_modify_tuple_by_cols(ctup, tupdesc, ncols, replaces, values,
- nulls);
- CatalogTupleUpdate(crel, &newtup->t_self, newtup);
- heap_freetuple(newtup);
+ newtup = heap_modify_tuple_by_cols(ctup, tupdesc, ncols, replaces, values,
+ nulls);
+ CatalogTupleUpdate(crel, &newtup->t_self, newtup);
+ heap_freetuple(newtup);
+ }
}
+ else if (inplace)
+ systable_inplace_update_cancel(inplace_state);
/* release the lock, consistent with vac_update_relstats() */
table_close(crel, RowExclusiveLock);
@@ -188,7 +226,7 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
Datum
pg_set_relation_stats(PG_FUNCTION_ARGS)
{
- relation_statistics_update(fcinfo, ERROR);
+ relation_statistics_update(fcinfo, ERROR, false);
PG_RETURN_VOID();
}
@@ -212,7 +250,7 @@ pg_clear_relation_stats(PG_FUNCTION_ARGS)
newfcinfo->args[3].value = DEFAULT_RELALLVISIBLE;
newfcinfo->args[3].isnull = false;
- relation_statistics_update(newfcinfo, ERROR);
+ relation_statistics_update(newfcinfo, ERROR, false);
PG_RETURN_VOID();
}
@@ -230,7 +268,7 @@ pg_restore_relation_stats(PG_FUNCTION_ARGS)
relarginfo, WARNING))
result = false;
- if (!relation_statistics_update(positional_fcinfo, WARNING))
+ if (!relation_statistics_update(positional_fcinfo, WARNING, true))
result = false;
PG_RETURN_BOOL(result);
--
2.47.0
[text/x-patch] v33-0005-split-out-check_conn_options.patch (4.5K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/5-v33-0005-split-out-check_conn_options.patch)
download | inline diff:
From 0d01cf8eece54da540f221e7d3e93279290fd573 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:20:07 -0500
Subject: [PATCH v33 05/11] split out check_conn_options
---
src/bin/scripts/vacuumdb.c | 103 ++++++++++++++++++++-----------------
1 file changed, 57 insertions(+), 46 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index d07ab7d67e..7b97a9428a 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -10,6 +10,7 @@
*-------------------------------------------------------------------------
*/
+#include "libpq-fe.h"
#include "postgres_fe.h"
#include <limits.h>
@@ -459,55 +460,11 @@ escape_quotes(const char *src)
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
+ * Check connection options for compatibility with the connected database.
*/
static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
- PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
- SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
- bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
-
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
-
- conn = connectDatabase(cparams, progname, echo, false, true);
-
if (vacopts->disable_page_skipping && PQserverVersion(conn) < 90600)
{
PQfinish(conn);
@@ -575,6 +532,60 @@ vacuum_one_database(ConnParams *cparams,
/* skip_database_stats is used automatically if server supports it */
vacopts->skip_database_stats = (PQserverVersion(conn) >= 160000);
+}
+
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PQExpBufferData buf;
+ PQExpBufferData catalog_query;
+ PGresult *res;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ SimpleStringList dbtables = {NULL, NULL};
+ int i;
+ int ntups;
+ bool failed = false;
+ bool objects_listed = false;
+ const char *initcmd;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
if (!quiet)
{
--
2.47.0
[text/x-patch] v33-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch (34.6K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/6-v33-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From 789830b90b4b4ac7edaa59555fd80b5e7c1fa5ee Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v33 01/11] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute. These
statements will restore the statistics of the current system onto the
destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, staistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index and Materialized View
statistics are dumped in the post-data section.
---
src/bin/pg_dump/pg_backup.h | 4 +
src/bin/pg_dump/pg_backup_archiver.c | 5 +
src/bin/pg_dump/pg_dump.c | 375 ++++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 8 +
src/bin/pg_dump/pg_dump_sort.c | 5 +
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 18 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
doc/src/sgml/ref/pg_dump.sgml | 36 ++-
doc/src/sgml/ref/pg_restore.sgml | 31 ++-
10 files changed, 489 insertions(+), 16 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b2..8fbb39d399 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -113,6 +113,7 @@ typedef struct _restoreOptions
int no_publications; /* Skip publication entries */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +161,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -182,6 +184,7 @@ typedef struct _dumpOptions
int no_security_labels;
int no_publications;
int no_subscriptions;
+ int no_statistics;
int no_toast_compression;
int no_unlogged_table_data;
int serializable_deferrable;
@@ -208,6 +211,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 33182d5b44..af8bb6bd12 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -2962,6 +2962,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's a stats dump, maybe ignore it */
+ if (ropt->no_statistics && strcmp(te->desc, "STATISTICS") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +2995,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index add7f16c90..40f75130b2 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -430,6 +430,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -466,6 +467,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -494,6 +496,7 @@ main(int argc, char **argv)
{"no-comments", no_argument, &dopt.no_comments, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -539,7 +542,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:PRsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -613,6 +616,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -784,6 +791,13 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -798,8 +812,20 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
+
+ if (statistics_only)
+ /* stats are only thing wanted */
+ dopt.dumpStatistics = true;
+ else if (dopt.no_statistics)
+ /* stats specifically excluded */
+ dopt.dumpStatistics = false;
+ else if (dopt.binary_upgrade)
+ /* binary upgrade and not specifically excluded */
+ dopt.dumpStatistics = true;
+ else
+ dopt.dumpStatistics = (!data_only && !schema_only);
/*
* --inserts are already implied above if --column-inserts or
@@ -1099,6 +1125,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1177,7 +1204,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1190,11 +1217,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1221,6 +1249,7 @@ help(const char *progname)
printf(_(" --no-comments do not dump comment commands\n"));
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6780,6 +6809,42 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7157,6 +7222,7 @@ getTables(Archive *fout, int *numTables)
/* Tables have data */
tblinfo[i].dobj.components |= DUMP_COMPONENT_DATA;
+ tblinfo[i].dobj.components |= DUMP_COMPONENT_STATISTICS;
/* Mark whether table has an ACL */
if (!PQgetisnull(res, i, i_relacl))
@@ -7205,6 +7271,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7649,11 +7717,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7676,7 +7747,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7710,6 +7788,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10297,6 +10377,276 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * get statistics dump section, which depends on the parent object type
+ *
+ * objects created in SECTION_PRE_DATA have stats in SECTION_DATA
+ * objects created in SECTION_POST_DATA have stats in SECTION_POST_DATA
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+
+ if ((rsinfo->relkind == RELKIND_MATVIEW) ||
+ (rsinfo->relkind == RELKIND_INDEX) ||
+ (rsinfo->relkind == RELKIND_PARTITIONED_INDEX))
+ return SECTION_POST_DATA;
+
+ return SECTION_DATA;
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = dobj->dependencies,
+ .nDeps = dobj->nDeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10745,6 +11095,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -17173,6 +17526,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo)
free(indstatvalsarray);
}
+ /* Comments and stats share same .dep */
+
/* Dump Index Comments */
if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT)
dumpComment(fout, "INDEX", qindxname,
@@ -18960,6 +19315,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index d65f558565..ef555e9178 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -84,6 +84,7 @@ typedef enum
DO_PUBLICATION_TABLE_IN_SCHEMA,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
+ DO_REL_STATS,
} DumpableObjectType;
/*
@@ -101,6 +102,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -421,6 +423,12 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'v', 'c', etc */
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index 3c8f2eb808..989d20aa27 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -1500,6 +1500,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 9a04e51c81..62e2766c09 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 88ae39d938..355f0439da 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,7 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -74,6 +75,7 @@ main(int argc, char **argv)
static int no_publications = 0;
static int no_security_labels = 0;
static int no_subscriptions = 0;
+ static int no_statistics = 0;
static int strict_names = 0;
bool data_only = false;
bool schema_only = false;
@@ -108,6 +110,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'P'},
/*
* the following options don't have an equivalent short option letter
@@ -128,6 +131,7 @@ main(int argc, char **argv)
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
{"filter", required_argument, NULL, 4},
+ {"no-statistics", no_argument, &no_statistics, 1},
{NULL, 0, NULL, 0}
};
@@ -271,6 +275,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +351,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +374,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpSchema = (!data_only && !statistics_only);
+ opts->dumpData = (!schema_only && !statistics_only);
+ opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +388,7 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index b9d13a0e1d..76ebf15f55 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d66e901f51..aa4785c612 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,8 +141,9 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or
+ <option>--statistics-only</option> is specified. The <option>-b</option>
switch is therefore only useful to add large objects to dumps
where a specific schema or table has been requested. Note that
large objects are considered data and therefore will be included when
@@ -516,10 +517,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +654,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -833,7 +847,8 @@ PostgreSQL documentation
though you do not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1098,6 +1113,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719..ff1441a243 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,20 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema or data.
+ </para>
+ <para>
+ (Do not confuse this with the <option>--schema</option> option, which
+ uses the word <quote>schema</quote> in a different meaning.)
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -723,6 +738,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
--
2.47.0
[text/x-patch] v33-0004-Consolidate-attribute-syscache-lookups-into-one-.patch (4.4K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/7-v33-0004-Consolidate-attribute-syscache-lookups-into-one-.patch)
download | inline diff:
From 0750f8c4fa8d5144857d7dfe2117d1414cc10b56 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 14 Nov 2024 03:53:33 -0500
Subject: [PATCH v33 04/11] Consolidate attribute syscache lookups into one
call by name.
Previously we were doing one lookup by attname and one lookup by attnum,
which seems wasteful.
---
src/backend/statistics/attribute_stats.c | 55 +++++++++++-------------
1 file changed, 24 insertions(+), 31 deletions(-)
diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c
index b97ba7b0c0..6393783f8e 100644
--- a/src/backend/statistics/attribute_stats.c
+++ b/src/backend/statistics/attribute_stats.c
@@ -78,8 +78,8 @@ static struct StatsArgInfo attarginfo[] =
static bool attribute_statistics_update(FunctionCallInfo fcinfo, int elevel);
static Node *get_attr_expr(Relation rel, int attnum);
-static void get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
- Oid *atttypid, int32 *atttypmod,
+static void get_attr_stat_type(Oid reloid, Name attname, int elevel,
+ AttrNumber *attnum, Oid *atttypid, int32 *atttypmod,
char *atttyptype, Oid *atttypcoll,
Oid *eq_opr, Oid *lt_opr);
static bool get_elem_stat_type(Oid atttypid, char atttyptype, int elevel,
@@ -166,23 +166,16 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel)
stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG);
attname = PG_GETARG_NAME(ATTNAME_ARG);
- attnum = get_attnum(reloid, NameStr(*attname));
-
- if (attnum < 0)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("cannot modify statistics on system column \"%s\"",
- NameStr(*attname))));
-
- if (attnum == InvalidAttrNumber)
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("column \"%s\" of relation \"%s\" does not exist",
- NameStr(*attname), get_rel_name(reloid))));
stats_check_required_arg(fcinfo, attarginfo, INHERITED_ARG);
inherited = PG_GETARG_BOOL(INHERITED_ARG);
+ /* derive type information from attribute */
+ get_attr_stat_type(reloid, attname, elevel,
+ &attnum, &atttypid, &atttypmod,
+ &atttyptype, &atttypcoll,
+ &eq_opr, <_opr);
+
/*
* Check argument sanity. If some arguments are unusable, emit at elevel
* and set the corresponding argument to NULL in fcinfo.
@@ -232,12 +225,6 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel)
result = false;
}
- /* derive information from attribute */
- get_attr_stat_type(reloid, attnum, elevel,
- &atttypid, &atttypmod,
- &atttyptype, &atttypcoll,
- &eq_opr, <_opr);
-
/* if needed, derive element type */
if (do_mcelem || do_dechist)
{
@@ -503,8 +490,8 @@ get_attr_expr(Relation rel, int attnum)
* Derive type information from the attribute.
*/
static void
-get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
- Oid *atttypid, int32 *atttypmod,
+get_attr_stat_type(Oid reloid, Name attname, int elevel,
+ AttrNumber *attnum, Oid *atttypid, int32 *atttypmod,
char *atttyptype, Oid *atttypcoll,
Oid *eq_opr, Oid *lt_opr)
{
@@ -514,24 +501,30 @@ get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
Node *expr;
TypeCacheEntry *typcache;
- atup = SearchSysCache2(ATTNUM, ObjectIdGetDatum(reloid),
- Int16GetDatum(attnum));
+ atup = SearchSysCacheAttName(reloid, NameStr(*attname));
- /* Attribute not found */
+ /* Attribute not found or is dropped */
if (!HeapTupleIsValid(atup))
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("attribute %d of relation \"%s\" does not exist",
- attnum, RelationGetRelationName(rel))));
+ errmsg("column \"%s\" of relation \"%s\" does not exist",
+ NameStr(*attname), get_rel_name(reloid))));
attr = (Form_pg_attribute) GETSTRUCT(atup);
- if (attr->attisdropped)
+ if (attr->attnum < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot modify statistics on system column \"%s\"",
+ NameStr(*attname))));
+
+ if (attr->attnum == InvalidAttrNumber)
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("attribute %d of relation \"%s\" does not exist",
- attnum, RelationGetRelationName(rel))));
+ errmsg("column \"%s\" of relation \"%s\" does not exist",
+ NameStr(*attname), get_rel_name(reloid))));
+ *attnum = attr->attnum;
expr = get_attr_expr(rel, attr->attnum);
/*
--
2.47.0
[text/x-patch] v33-0006-split-out-print_processing_notice.patch (2.4K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/8-v33-0006-split-out-print_processing_notice.patch)
download | inline diff:
From 041dd15f564088ed6b2df7885af7ffdcd71cb554 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:46:51 -0500
Subject: [PATCH v33 06/11] split out print_processing_notice
---
src/bin/scripts/vacuumdb.c | 41 +++++++++++++++++++++++---------------
1 file changed, 25 insertions(+), 16 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 7b97a9428a..e9946f79b2 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -535,6 +535,30 @@ check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
}
+/*
+ * print the processing notice for a database.
+ */
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet)
+{
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ if (!quiet)
+ {
+ if (stage != ANALYZE_NO_STAGE)
+ printf(_("%s: processing database \"%s\": %s\n"),
+ progname, PQdb(conn), _(stage_messages[stage]));
+ else
+ printf(_("%s: vacuuming database \"%s\"\n"),
+ progname, PQdb(conn));
+ fflush(stdout);
+ }
+}
+
/*
* vacuum_one_database
*
@@ -574,11 +598,6 @@ vacuum_one_database(ConnParams *cparams,
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
"RESET default_statistics_target;"
};
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
@@ -586,17 +605,7 @@ vacuum_one_database(ConnParams *cparams,
conn = connectDatabase(cparams, progname, echo, false, true);
check_conn_options(conn, vacopts);
-
- if (!quiet)
- {
- if (stage != ANALYZE_NO_STAGE)
- printf(_("%s: processing database \"%s\": %s\n"),
- progname, PQdb(conn), _(stage_messages[stage]));
- else
- printf(_("%s: vacuuming database \"%s\"\n"),
- progname, PQdb(conn));
- fflush(stdout);
- }
+ print_processing_notice(conn, stage, progname, quiet);
/*
* Prepare the list of tables to process by querying the catalogs.
--
2.47.0
[text/x-patch] v33-0007-split-out-generate_catalog_list.patch (7.1K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/9-v33-0007-split-out-generate_catalog_list.patch)
download | inline diff:
From ccb3412a776e5df2dae1f3f1d2b7f708b310f2af Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 16:15:16 -0500
Subject: [PATCH v33 07/11] split out generate_catalog_list
---
src/bin/scripts/vacuumdb.c | 176 +++++++++++++++++++++----------------
1 file changed, 99 insertions(+), 77 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index e9946f79b2..36f4796db0 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -560,67 +560,38 @@ print_processing_notice(PGconn *conn, int stage, const char *progname, bool quie
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
- */
-static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+ * Prepare the list of tables to process by querying the catalogs.
+ *
+ * Since we execute the constructed query with the default search_path
+ * (which could be unsafe), everything in this query MUST be fully
+ * qualified.
+ *
+ * First, build a WITH clause for the catalog query if any tables were
+ * specified, with a set of values made of relation names and their
+ * optional set of columns. This is used to match any provided column
+ * lists with the generated qualified identifiers and to filter for the
+ * tables provided via --table. If a listed table does not exist, the
+ * catalog query will fail.
+ */
+static SimpleStringList *
+generate_catalog_list(PGconn *conn,
+ vacuumingOptions *vacopts,
+ SimpleStringList *objects,
+ bool echo,
+ int *ntups)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
+ PQExpBufferData buf;
+ SimpleStringList *dbtables;
SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
+ PGresult *res;
+ int i;
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+ dbtables = palloc(sizeof(SimpleStringList));
+ dbtables->head = NULL;
+ dbtables->tail = NULL;
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- /*
- * Prepare the list of tables to process by querying the catalogs.
- *
- * Since we execute the constructed query with the default search_path
- * (which could be unsafe), everything in this query MUST be fully
- * qualified.
- *
- * First, build a WITH clause for the catalog query if any tables were
- * specified, with a set of values made of relation names and their
- * optional set of columns. This is used to match any provided column
- * lists with the generated qualified identifiers and to filter for the
- * tables provided via --table. If a listed table does not exist, the
- * catalog query will fail.
- */
initPQExpBuffer(&catalog_query);
for (cell = objects ? objects->head : NULL; cell; cell = cell->next)
{
@@ -771,40 +742,91 @@ vacuum_one_database(ConnParams *cparams,
appendPQExpBufferStr(&catalog_query, " ORDER BY c.relpages DESC;");
executeCommand(conn, "RESET search_path;", echo);
res = executeQuery(conn, catalog_query.data, echo);
+ *ntups = PQntuples(res);
termPQExpBuffer(&catalog_query);
PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo));
- /*
- * If no rows are returned, there are no matching tables, so we are done.
- */
- ntups = PQntuples(res);
- if (ntups == 0)
- {
- PQclear(res);
- PQfinish(conn);
- return;
- }
-
/*
* Build qualified identifiers for each table, including the column list
* if given.
*/
- initPQExpBuffer(&buf);
- for (i = 0; i < ntups; i++)
+ if (*ntups > 0)
{
- appendPQExpBufferStr(&buf,
- fmtQualifiedId(PQgetvalue(res, i, 1),
- PQgetvalue(res, i, 0)));
+ initPQExpBuffer(&buf);
+ for (i = 0; i < *ntups; i++)
+ {
+ appendPQExpBufferStr(&buf,
+ fmtQualifiedId(PQgetvalue(res, i, 1),
+ PQgetvalue(res, i, 0)));
- if (objects_listed && !PQgetisnull(res, i, 2))
- appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
+ if (objects_listed && !PQgetisnull(res, i, 2))
+ appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
- simple_string_list_append(&dbtables, buf.data);
- resetPQExpBuffer(&buf);
+ simple_string_list_append(dbtables, buf.data);
+ resetPQExpBuffer(&buf);
+ }
+ termPQExpBuffer(&buf);
}
- termPQExpBuffer(&buf);
PQclear(res);
+ return dbtables;
+}
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ int ntups;
+ bool failed = false;
+ const char *initcmd;
+ SimpleStringList *dbtables;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+
+ dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
+
+ /*
+ * If no rows are returned, there are no matching tables, so we are done.
+ */
+ if (ntups == 0)
+ {
+ PQfinish(conn);
+ return;
+ }
+
+
/*
* Ensure concurrentCons is sane. If there are more connections than
* vacuumable relations, we don't need to use them all.
@@ -837,7 +859,7 @@ vacuum_one_database(ConnParams *cparams,
initPQExpBuffer(&sql);
- cell = dbtables.head;
+ cell = dbtables->head;
do
{
const char *tabname = cell->val;
--
2.47.0
[text/x-patch] v33-0009-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch (1.5K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/10-v33-0009-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch)
download | inline diff:
From e924cc42fa8e2ae3e479f125cfaae2d82c193dbf Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 7 Nov 2024 18:06:52 -0500
Subject: [PATCH v33 09/11] Add issues_sql_unlike, opposite of issues_sql_like
This is the same as issues_sql_like(), but the specified text is
prohibited from being in the output rather than required.
This became necessary to test that a command-line filter was in fact
filtering out certain output that a prior test required.
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 25 ++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 508e5e3917..1281071479 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2799,6 +2799,31 @@ sub issues_sql_like
}
=pod
+=item $node->issues_sql_unlike(cmd, prohibited_sql, test_name)
+
+Run a command on the node, then verify that $prohibited_sql does not appear
+in the server log file.
+
+=cut
+
+sub issues_sql_unlike
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+ my ($self, $cmd, $prohibited_sql, $test_name) = @_;
+
+ local %ENV = $self->_get_env();
+
+ my $log_location = -s $self->logfile;
+
+ my $result = PostgreSQL::Test::Utils::run_log($cmd);
+ ok($result, "@$cmd exit code 0");
+ my $log =
+ PostgreSQL::Test::Utils::slurp_file($self->logfile, $log_location);
+ unlike($log, $prohibited_sql, "$test_name: SQL found in server log");
+ return;
+}
+
=item $node->log_content()
--
2.47.0
[text/x-patch] v33-0008-preserve-catalog-lists-across-staged-runs.patch (6.7K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/11-v33-0008-preserve-catalog-lists-across-staged-runs.patch)
download | inline diff:
From 975dd88313e88ae8d579b1e0cd68d7944cb6a391 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 21:30:49 -0500
Subject: [PATCH v33 08/11] preserve catalog lists across staged runs
---
src/bin/scripts/vacuumdb.c | 113 ++++++++++++++++++++++++++-----------
1 file changed, 80 insertions(+), 33 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 36f4796db0..b13f3c4224 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -68,6 +68,9 @@ static void vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet);
static void vacuum_all_databases(ConnParams *cparams,
@@ -83,6 +86,14 @@ static void prepare_vacuum_command(PQExpBuffer sql, int serverVersion,
static void run_vacuum_command(PGconn *conn, const char *sql, bool echo,
const char *table);
+static void check_conn_options(PGconn *conn, vacuumingOptions *vacopts);
+
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet);
+
+static SimpleStringList * generate_catalog_list(PGconn *conn, vacuumingOptions *vacopts,
+ SimpleStringList *objects, bool echo, int *ntups);
+
static void help(const char *progname);
void check_objfilter(void);
@@ -386,6 +397,11 @@ main(int argc, char *argv[])
}
else
{
+ PGconn *conn;
+ int ntup;
+ SimpleStringList *found_objects;
+ int stage;
+
if (dbname == NULL)
{
if (getenv("PGDATABASE"))
@@ -397,25 +413,37 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ check_conn_options(conn, &vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
if (analyze_in_stages)
{
- int stage;
-
for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
{
- vacuum_one_database(&cparams, &vacopts,
- stage,
+ /* the last pass disconnected the conn */
+ if (stage > 0)
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
}
else
- vacuum_one_database(&cparams, &vacopts,
- ANALYZE_NO_STAGE,
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
@@ -791,16 +819,17 @@ vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet)
{
PQExpBufferData sql;
- PGconn *conn;
SimpleStringListCell *cell;
ParallelSlotArray *sa;
- int ntups;
bool failed = false;
const char *initcmd;
- SimpleStringList *dbtables;
+
const char *stage_commands[] = {
"SET default_statistics_target=1; SET vacuum_cost_delay=0;",
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
@@ -810,13 +839,6 @@ vacuum_one_database(ConnParams *cparams,
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
-
/*
* If no rows are returned, there are no matching tables, so we are done.
*/
@@ -928,7 +950,7 @@ finish:
}
/*
- * Vacuum/analyze all connectable databases.
+ * Vacuum/analyze all ccparams->override_dbname = PQgetvalue(result, i, 0);onnectable databases.
*
* In analyze-in-stages mode, we process all databases in one stage before
* moving on to the next stage. That ensure minimal stats are available
@@ -944,8 +966,13 @@ vacuum_all_databases(ConnParams *cparams,
{
PGconn *conn;
PGresult *result;
- int stage;
int i;
+ int stage;
+
+ SimpleStringList **found_objects;
+ int *num_tuples;
+
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
@@ -953,7 +980,33 @@ vacuum_all_databases(ConnParams *cparams,
echo);
PQfinish(conn);
- if (analyze_in_stages)
+ /*
+ * connect to each database, check validity of options,
+ * build the list of found objects per database,
+ * and run the first/only vacuum stage
+ */
+ found_objects = palloc(PQntuples(result) * sizeof(SimpleStringList *));
+ num_tuples = palloc(PQntuples(result) * sizeof (int));
+
+ for (i = 0; i < PQntuples(result); i++)
+ {
+ cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects[i] = generate_catalog_list(conn, vacopts, objects, echo, &num_tuples[i]);
+
+ vacuum_one_database(cparams, vacopts,
+ stage,
+ objects,
+ concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
+ progname, echo, quiet);
+ }
+
+ if (stage != ANALYZE_NO_STAGE)
{
/*
* When analyzing all databases in stages, we analyze them all in the
@@ -963,35 +1016,29 @@ vacuum_all_databases(ConnParams *cparams,
* This means we establish several times as many connections, but
* that's a secondary consideration.
*/
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
for (i = 0; i < PQntuples(result); i++)
{
cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(cparams, vacopts,
stage,
objects,
concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
progname, echo, quiet);
}
}
}
- else
- {
- for (i = 0; i < PQntuples(result); i++)
- {
- cparams->override_dbname = PQgetvalue(result, i, 0);
-
- vacuum_one_database(cparams, vacopts,
- ANALYZE_NO_STAGE,
- objects,
- concurrentCons,
- progname, echo, quiet);
- }
- }
PQclear(result);
+ pfree(found_objects);
+ pfree(num_tuples);
}
/*
--
2.47.0
[text/x-patch] v33-0010-Add-force-analyze-to-vacuumdb.patch (12.1K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/12-v33-0010-Add-force-analyze-to-vacuumdb.patch)
download | inline diff:
From ae94d0a1f6af01a0816d9515d6b03d1de9e0596e Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Fri, 8 Nov 2024 12:27:50 -0500
Subject: [PATCH v33 10/11] Add --force-analyze to vacuumdb.
The vacuumdb options of --analyze-in-stages and --analyze-only are often
used after a restore from a dump or a pg_upgrade to quickly rebuild
stats on a databse.
However, now that stats are imported in most (but not all) cases,
running either of these commands will be at least partially redundant,
and will overwrite the stats that were just imported, which is a big
POLA violation.
We could add a new option such as --analyze-missing-in-stages, but that
wouldn't help the userbase that grown accustomed to running
--analyze-in-stages after an upgrade.
The least-bad option to handle the situation is to change the behavior
of --analyze-only and --analyze-in-stages to only analyze tables which
were missing stats before the vacuumdb started, but offer the
--force-analyze flag to restore the old behavior for those who truly
wanted it.
---
src/bin/scripts/t/100_vacuumdb.pl | 6 +-
src/bin/scripts/vacuumdb.c | 91 ++++++++++++++++++++++++-------
doc/src/sgml/ref/vacuumdb.sgml | 48 ++++++++++++++++
3 files changed, 125 insertions(+), 20 deletions(-)
diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl
index 1a2bcb4959..2d669391fe 100644
--- a/src/bin/scripts/t/100_vacuumdb.pl
+++ b/src/bin/scripts/t/100_vacuumdb.pl
@@ -127,9 +127,13 @@ $node->issues_sql_like(
qr/statement: VACUUM \(SKIP_DATABASE_STATS, ANALYZE\) public.vactable\(a, b\);/,
'vacuumdb --analyze with complete column list');
$node->issues_sql_like(
+ [ 'vacuumdb', '--analyze-only', '--force-analyze', '--table', 'vactable(b)', 'postgres' ],
+ qr/statement: ANALYZE public.vactable\(b\);/,
+ 'vacuumdb --analyze-only --force-analyze with partial column list');
+$node->issues_sql_unlike(
[ 'vacuumdb', '--analyze-only', '--table', 'vactable(b)', 'postgres' ],
qr/statement: ANALYZE public.vactable\(b\);/,
- 'vacuumdb --analyze-only with partial column list');
+ 'vacuumdb --analyze-only --force-analyze with partial column list skipping vacuumed tables');
$node->command_checks_all(
[ 'vacuumdb', '--analyze', '--table', 'vacview', 'postgres' ],
0,
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index b13f3c4224..1aa5c46af5 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -25,6 +25,7 @@
#include "fe_utils/query_utils.h"
#include "fe_utils/simple_list.h"
#include "fe_utils/string_utils.h"
+#include "pqexpbuffer.h"
/* vacuum options controlled by user flags */
@@ -47,6 +48,8 @@ typedef struct vacuumingOptions
bool process_main;
bool process_toast;
bool skip_database_stats;
+ bool analyze_in_stages;
+ bool force_analyze;
char *buffer_usage_limit;
} vacuumingOptions;
@@ -75,7 +78,6 @@ static void vacuum_one_database(ConnParams *cparams,
static void vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet);
@@ -140,6 +142,7 @@ main(int argc, char *argv[])
{"no-process-toast", no_argument, NULL, 11},
{"no-process-main", no_argument, NULL, 12},
{"buffer-usage-limit", required_argument, NULL, 13},
+ {"force-analyze", no_argument, NULL, 14},
{NULL, 0, NULL, 0}
};
@@ -156,7 +159,6 @@ main(int argc, char *argv[])
bool echo = false;
bool quiet = false;
vacuumingOptions vacopts;
- bool analyze_in_stages = false;
SimpleStringList objects = {NULL, NULL};
int concurrentCons = 1;
int tbl_count = 0;
@@ -170,6 +172,8 @@ main(int argc, char *argv[])
vacopts.do_truncate = true;
vacopts.process_main = true;
vacopts.process_toast = true;
+ vacopts.force_analyze = false;
+ vacopts.analyze_in_stages = false;
pg_logging_init(argv[0]);
progname = get_progname(argv[0]);
@@ -251,7 +255,7 @@ main(int argc, char *argv[])
maintenance_db = pg_strdup(optarg);
break;
case 3:
- analyze_in_stages = vacopts.analyze_only = true;
+ vacopts.analyze_in_stages = vacopts.analyze_only = true;
break;
case 4:
vacopts.disable_page_skipping = true;
@@ -287,6 +291,9 @@ main(int argc, char *argv[])
case 13:
vacopts.buffer_usage_limit = escape_quotes(optarg);
break;
+ case 14:
+ vacopts.force_analyze = true;
+ break;
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -372,6 +379,14 @@ main(int argc, char *argv[])
pg_fatal("cannot use the \"%s\" option with the \"%s\" option",
"buffer-usage-limit", "full");
+ /*
+ * --force-analyze is only valid when used with --analyze-only, -analyze,
+ * or --analyze-in-stages
+ */
+ if (vacopts.force_analyze && !vacopts.analyze_only && !vacopts.analyze_in_stages)
+ pg_fatal("can only use the \"%s\" option with \"%s\" or \"%s\"",
+ "--force-analyze", "-Z/--analyze-only", "--analyze-in-stages");
+
/* fill cparams except for dbname, which is set below */
cparams.pghost = host;
cparams.pgport = port;
@@ -390,7 +405,6 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
vacuum_all_databases(&cparams, &vacopts,
- analyze_in_stages,
&objects,
concurrentCons,
progname, echo, quiet);
@@ -413,20 +427,27 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts.analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectDatabase(&cparams, progname, echo, false, true);
check_conn_options(conn, &vacopts);
print_processing_notice(conn, stage, progname, quiet);
found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
+ vacuum_one_database(&cparams, &vacopts, stage,
+ &objects,
+ concurrentCons,
+ conn,
+ found_objects,
+ ntup,
+ progname, echo, quiet);
- if (analyze_in_stages)
+ if (stage != ANALYZE_NO_STAGE)
{
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
/* the last pass disconnected the conn */
- if (stage > 0)
- conn = connectDatabase(&cparams, progname, echo, false, true);
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(&cparams, &vacopts, stage,
&objects,
@@ -437,14 +458,6 @@ main(int argc, char *argv[])
progname, echo, quiet);
}
}
- else
- vacuum_one_database(&cparams, &vacopts, stage,
- &objects,
- concurrentCons,
- conn,
- found_objects,
- ntup,
- progname, echo, quiet);
}
exit(0);
@@ -763,6 +776,47 @@ generate_catalog_list(PGconn *conn,
vacopts->min_mxid_age);
}
+ /*
+ * If this query is for an analyze-only or analyze-in-stages, two
+ * upgrade-centric operations, and force-analyze is NOT set, then
+ * exclude any relations that already have their full compliment
+ * of attribute stats and extended stats.
+ *
+ *
+ */
+ if ((vacopts->analyze_only || vacopts->analyze_in_stages) &&
+ !vacopts->force_analyze)
+ {
+ /*
+ * The pg_class in question has no pg_statistic rows representing
+ * user-visible columns that lack a corresponding pg_statitic row.
+ * Currently no differentiation is made for whether the
+ * pg_statistic.stainherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_attribute AS a\n"
+ " WHERE a.attrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND a.attnum OPERATOR(pg_catalog.>) 0 AND NOT a.attisdropped\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic AS s\n"
+ " WHERE s.starelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND s.staattnum OPERATOR(pg_catalog.=) a.attnum))\n");
+
+ /*
+ * The pg_class entry has no pg_statistic_ext rows that lack a corresponding
+ * pg_statistic_ext_data row. Currently no differentiation is made for whether
+ * pg_statistic_exta_data.stxdinherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext AS e\n"
+ " WHERE e.stxrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext_data AS d\n"
+ " WHERE d.stxoid OPERATOR(pg_catalog.=) e.oid))\n");
+ }
+
/*
* Execute the catalog query. We use the default search_path for this
* query for consistency with table lookups done elsewhere by the user.
@@ -959,7 +1013,6 @@ finish:
static void
vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet)
@@ -972,7 +1025,7 @@ vacuum_all_databases(ConnParams *cparams,
SimpleStringList **found_objects;
int *num_tuples;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts->analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml
index 66fccb30a2..00ec927606 100644
--- a/doc/src/sgml/ref/vacuumdb.sgml
+++ b/doc/src/sgml/ref/vacuumdb.sgml
@@ -425,6 +425,12 @@ PostgreSQL documentation
<para>
Only calculate statistics for use by the optimizer (no vacuum).
</para>
+ <para>
+ By default, this operation excludes relations that already have
+ statistics generated. If the option <option>--force-analyze</option>
+ is also specified, then relations with existing stastistics are not
+ excluded.
+ </para>
</listitem>
</varlistentry>
@@ -439,6 +445,47 @@ PostgreSQL documentation
to produce usable statistics faster, and subsequent stages build the
full statistics.
</para>
+ <para>
+ This option is intended to be run after a <command>pg_upgrade</command>
+ to generate statistics for relations that have no stistatics or incomplete
+ statistics (such as those with extended statistics objects, which are not
+ imported on upgrade).
+ </para>
+ <para>
+ If the option <option>--force-analyze</option> is also specified, then
+ relations with existing stastistics are not excluded.
+ This option is only useful to analyze a database that currently has
+ no statistics or has wholly incorrect ones, such as if it is newly
+ populated from a restored dump.
+ Be aware that running with this option combinationin a database with
+ existing statistics may cause the query optimizer choices to become
+ transiently worse due to the low statistics targets of the early
+ stages.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--force-analyze</option></term>
+ <listitem>
+ <para>
+ This option can only be used if either <option>--analyze-only</option>
+ or <option>--analyze-in-stages</option> is specified. It modifies those
+ options to not filter out relations that already have statistics.
+ </para>
+ <para>
+
+ Only calculate statistics for use by the optimizer (no vacuum),
+ like <option>--analyze-only</option>. Run three
+ stages of analyze; the first stage uses the lowest possible statistics
+ target (see <xref linkend="guc-default-statistics-target"/>)
+ to produce usable statistics faster, and subsequent stages build the
+ full statistics.
+ </para>
+
+ <para>
+ This option was created
+ </para>
<para>
This option is only useful to analyze a database that currently has
@@ -452,6 +499,7 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+
<varlistentry>
<term><option>-?</option></term>
<term><option>--help</option></term>
--
2.47.0
[text/x-patch] v33-0011-Enable-pg_clear_relation_stats-to-handle-differe.patch (4.3K, ../../CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com/13-v33-0011-Enable-pg_clear_relation_stats-to-handle-differe.patch)
download | inline diff:
From 8dafe8543f0c5e6204f8f8161e6c349b6a022d33 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Mon, 4 Nov 2024 16:21:39 -0500
Subject: [PATCH v33 11/11] Enable pg_clear_relation_stats to handle different
default relpages.
If it comes to pass that relpages has the default of -1.0 for
partitioned tables (and indexes), then this patch will handle that.
---
src/backend/statistics/relation_stats.c | 47 +++++++++++++------------
1 file changed, 24 insertions(+), 23 deletions(-)
diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c
index 24f646048c..e27fe79406 100644
--- a/src/backend/statistics/relation_stats.c
+++ b/src/backend/statistics/relation_stats.c
@@ -19,15 +19,12 @@
#include "access/heapam.h"
#include "catalog/indexing.h"
+#include "catalog/pg_class_d.h"
#include "statistics/stat_utils.h"
#include "utils/fmgrprotos.h"
#include "utils/syscache.h"
#include "utils/fmgroids.h"
-#define DEFAULT_RELPAGES Int32GetDatum(0)
-#define DEFAULT_RELTUPLES Float4GetDatum(-1.0)
-#define DEFAULT_RELALLVISIBLE Int32GetDatum(0)
-
/*
* Positional argument numbers, names, and types for
* relation_statistics_update().
@@ -51,14 +48,11 @@ static struct StatsArgInfo relarginfo[] =
[NUM_RELATION_STATS_ARGS] = {0}
};
-static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel,
- bool inplace);
-
/*
* Internal function for modifying statistics for a relation.
*/
static bool
-relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace)
+relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace, bool clear)
{
Oid reloid;
Relation crel;
@@ -116,9 +110,19 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace)
pgcform = (Form_pg_class) GETSTRUCT(ctup);
/* relpages */
- if (!PG_ARGISNULL(RELPAGES_ARG))
+ if (!PG_ARGISNULL(RELPAGES_ARG) || clear)
{
- int32 relpages = PG_GETARG_INT32(RELPAGES_ARG);
+ int32 relpages;
+
+ if (clear)
+ /* relpages default varies by relkind */
+ if ((crel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) ||
+ (crel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX))
+ relpages = -1;
+ else
+ relpages = 0;
+ else
+ relpages = PG_GETARG_INT32(RELPAGES_ARG);
/*
* Partitioned tables may have relpages=-1. Note: for relations with
@@ -145,9 +149,9 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace)
}
}
- if (!PG_ARGISNULL(RELTUPLES_ARG))
+ if (!PG_ARGISNULL(RELTUPLES_ARG) || clear)
{
- float reltuples = PG_GETARG_FLOAT4(RELTUPLES_ARG);
+ float reltuples = (clear) ? -1.0 : PG_GETARG_FLOAT4(RELTUPLES_ARG);
if (reltuples < -1.0)
{
@@ -170,9 +174,9 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace)
}
- if (!PG_ARGISNULL(RELALLVISIBLE_ARG))
+ if (!PG_ARGISNULL(RELALLVISIBLE_ARG) || clear)
{
- int32 relallvisible = PG_GETARG_INT32(RELALLVISIBLE_ARG);
+ int32 relallvisible = (clear) ? 0 : PG_GETARG_INT32(RELALLVISIBLE_ARG);
if (relallvisible < 0)
{
@@ -226,7 +230,7 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace)
Datum
pg_set_relation_stats(PG_FUNCTION_ARGS)
{
- relation_statistics_update(fcinfo, ERROR, false);
+ relation_statistics_update(fcinfo, ERROR, false, false);
PG_RETURN_VOID();
}
@@ -243,14 +247,11 @@ pg_clear_relation_stats(PG_FUNCTION_ARGS)
newfcinfo->args[0].value = PG_GETARG_OID(0);
newfcinfo->args[0].isnull = PG_ARGISNULL(0);
- newfcinfo->args[1].value = DEFAULT_RELPAGES;
- newfcinfo->args[1].isnull = false;
- newfcinfo->args[2].value = DEFAULT_RELTUPLES;
- newfcinfo->args[2].isnull = false;
- newfcinfo->args[3].value = DEFAULT_RELALLVISIBLE;
- newfcinfo->args[3].isnull = false;
+ newfcinfo->args[1].isnull = true;
+ newfcinfo->args[2].isnull = true;
+ newfcinfo->args[3].isnull = true;
- relation_statistics_update(newfcinfo, ERROR, false);
+ relation_statistics_update(newfcinfo, ERROR, false, true);
PG_RETURN_VOID();
}
@@ -268,7 +269,7 @@ pg_restore_relation_stats(PG_FUNCTION_ARGS)
relarginfo, WARNING))
result = false;
- if (!relation_statistics_update(positional_fcinfo, WARNING, true))
+ if (!relation_statistics_update(positional_fcinfo, WARNING, true, false))
result = false;
PG_RETURN_BOOL(result);
--
2.47.0
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-11-27 05:08 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2024-12-07 00:44 ` Jeff Davis <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Jeff Davis @ 2024-12-07 00:44 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Wed, 2024-11-27 at 00:08 -0500, Corey Huinker wrote:
>
> 0003 - Re-enabling in-place updates because catalog bloat bad.
Attached is my version of this patch, which I intend to commit soon.
I added docs and tests, and I refactored a bit to check the arguments
first.
Also, I separated the mvcc and in-place paths, so that it was easier to
review that each one is following the right protocol.
Regards,
Jeff Davis
Attachments:
[text/x-patch] v33j-0001-Use-in-place-updates-for-pg_restore_relation_st.patch (13.8K, ../../[email protected]/2-v33j-0001-Use-in-place-updates-for-pg_restore_relation_st.patch)
download | inline diff:
From a41503b82469297797db78163426c40a7a86317f Mon Sep 17 00:00:00 2001
From: Jeff Davis <[email protected]>
Date: Fri, 6 Dec 2024 12:32:29 -0800
Subject: [PATCH v33j] Use in-place updates for pg_restore_relation_stats().
This matches the behavior of vac_update_relstats(), which is important
to avoid bloating pg_class.
Author: Corey Huinker
Discussion: https://postgr.es/m/CADkLM=fc3je+ufv3gsHqjjSSf+t8674RXpuXW62EL55MUEQd-g@mail.gmail.com
---
doc/src/sgml/func.sgml | 8 +
src/backend/statistics/relation_stats.c | 198 +++++++++++++--------
src/test/regress/expected/stats_import.out | 61 +++++++
src/test/regress/sql/stats_import.sql | 37 ++++
4 files changed, 233 insertions(+), 71 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 8b81106fa23..2c35252dc06 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -30175,6 +30175,14 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
function is to maintain a consistent function signature to avoid
errors when restoring statistics from previous versions.
</para>
+ <para>
+ To match the behavior of <xref linkend="sql-vacuum"/> and <xref
+ linkend="sql-analyze"/> when updating relation statistics,
+ <function>pg_restore_relation_stats()</function> does not follow MVCC
+ transactional semantics (see <xref linkend="mvcc"/>). New relation
+ statistics may be durable even if the transaction aborts, and the
+ changes are not isolated from other transactions.
+ </para>
<para>
Arguments are passed as pairs of <replaceable>argname</replaceable>
and <replaceable>argvalue</replaceable>, where
diff --git a/src/backend/statistics/relation_stats.c b/src/backend/statistics/relation_stats.c
index e619d5cf5b1..f84157b16ab 100644
--- a/src/backend/statistics/relation_stats.c
+++ b/src/backend/statistics/relation_stats.c
@@ -20,6 +20,7 @@
#include "access/heapam.h"
#include "catalog/indexing.h"
#include "statistics/stat_utils.h"
+#include "utils/fmgroids.h"
#include "utils/fmgrprotos.h"
#include "utils/syscache.h"
@@ -50,59 +51,28 @@ static struct StatsArgInfo relarginfo[] =
[NUM_RELATION_STATS_ARGS] = {0}
};
-static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel);
+static bool relation_statistics_update(FunctionCallInfo fcinfo, int elevel,
+ bool inplace);
/*
* Internal function for modifying statistics for a relation.
*/
static bool
-relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
+relation_statistics_update(FunctionCallInfo fcinfo, int elevel, bool inplace)
{
Oid reloid;
Relation crel;
- HeapTuple ctup;
- Form_pg_class pgcform;
- int replaces[3] = {0};
- Datum values[3] = {0};
- bool nulls[3] = {0};
- int ncols = 0;
- TupleDesc tupdesc;
+ int32 relpages = DEFAULT_RELPAGES;
+ bool update_relpages = false;
+ float reltuples = DEFAULT_RELTUPLES;
+ bool update_reltuples = false;
+ int32 relallvisible = DEFAULT_RELALLVISIBLE;
+ bool update_relallvisible = false;
bool result = true;
- stats_check_required_arg(fcinfo, relarginfo, RELATION_ARG);
- reloid = PG_GETARG_OID(RELATION_ARG);
-
- if (RecoveryInProgress())
- ereport(ERROR,
- (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
- errmsg("recovery is in progress"),
- errhint("Statistics cannot be modified during recovery.")));
-
- stats_lock_check_privileges(reloid);
-
- /*
- * Take RowExclusiveLock on pg_class, consistent with
- * vac_update_relstats().
- */
- crel = table_open(RelationRelationId, RowExclusiveLock);
-
- tupdesc = RelationGetDescr(crel);
- ctup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(reloid));
- if (!HeapTupleIsValid(ctup))
- {
- ereport(elevel,
- (errcode(ERRCODE_OBJECT_IN_USE),
- errmsg("pg_class entry for relid %u not found", reloid)));
- table_close(crel, RowExclusiveLock);
- return false;
- }
-
- pgcform = (Form_pg_class) GETSTRUCT(ctup);
-
- /* relpages */
if (!PG_ARGISNULL(RELPAGES_ARG))
{
- int32 relpages = PG_GETARG_INT32(RELPAGES_ARG);
+ relpages = PG_GETARG_INT32(RELPAGES_ARG);
/*
* Partitioned tables may have relpages=-1. Note: for relations with
@@ -116,17 +86,13 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
errmsg("relpages cannot be < -1")));
result = false;
}
- else if (relpages != pgcform->relpages)
- {
- replaces[ncols] = Anum_pg_class_relpages;
- values[ncols] = Int32GetDatum(relpages);
- ncols++;
- }
+ else
+ update_relpages = true;
}
if (!PG_ARGISNULL(RELTUPLES_ARG))
{
- float reltuples = PG_GETARG_FLOAT4(RELTUPLES_ARG);
+ reltuples = PG_GETARG_FLOAT4(RELTUPLES_ARG);
if (reltuples < -1.0)
{
@@ -135,18 +101,13 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
errmsg("reltuples cannot be < -1.0")));
result = false;
}
- else if (reltuples != pgcform->reltuples)
- {
- replaces[ncols] = Anum_pg_class_reltuples;
- values[ncols] = Float4GetDatum(reltuples);
- ncols++;
- }
-
+ else
+ update_reltuples = true;
}
if (!PG_ARGISNULL(RELALLVISIBLE_ARG))
{
- int32 relallvisible = PG_GETARG_INT32(RELALLVISIBLE_ARG);
+ relallvisible = PG_GETARG_INT32(RELALLVISIBLE_ARG);
if (relallvisible < 0)
{
@@ -155,23 +116,118 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
errmsg("relallvisible cannot be < 0")));
result = false;
}
- else if (relallvisible != pgcform->relallvisible)
+ else
+ update_relallvisible = true;
+ }
+
+ stats_check_required_arg(fcinfo, relarginfo, RELATION_ARG);
+ reloid = PG_GETARG_OID(RELATION_ARG);
+
+ if (RecoveryInProgress())
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("recovery is in progress"),
+ errhint("Statistics cannot be modified during recovery.")));
+
+ stats_lock_check_privileges(reloid);
+
+ /*
+ * Take RowExclusiveLock on pg_class, consistent with
+ * vac_update_relstats().
+ */
+ crel = table_open(RelationRelationId, RowExclusiveLock);
+
+ if (inplace)
+ {
+ HeapTuple ctup = NULL;
+ ScanKeyData key[1];
+ Form_pg_class pgcform;
+ void *inplace_state = NULL;
+ bool dirty = false;
+
+ ScanKeyInit(&key[0], Anum_pg_class_oid, BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(reloid));
+ systable_inplace_update_begin(crel, ClassOidIndexId, true, NULL, 1, key,
+ &ctup, &inplace_state);
+ if (!HeapTupleIsValid(ctup))
+ elog(ERROR, "pg_class entry for relid %u vanished while updating statistics",
+ reloid);
+ pgcform = (Form_pg_class) GETSTRUCT(ctup);
+
+ if (update_relpages && relpages != pgcform->relpages)
+ {
+ pgcform->relpages = relpages;
+ dirty = true;
+ }
+ if (update_reltuples && pgcform->reltuples != reltuples)
{
- replaces[ncols] = Anum_pg_class_relallvisible;
- values[ncols] = Int32GetDatum(relallvisible);
- ncols++;
+ pgcform->reltuples = reltuples;
+ dirty = true;
+ }
+ if (update_relallvisible && pgcform->relallvisible != relallvisible)
+ {
+ pgcform->relallvisible = relallvisible;
+ dirty = true;
}
- }
- /* only update pg_class if there is a meaningful change */
- if (ncols > 0)
+ if (dirty)
+ systable_inplace_update_finish(inplace_state, ctup);
+ else
+ systable_inplace_update_cancel(inplace_state);
+ }
+ else
{
- HeapTuple newtup;
+ TupleDesc tupdesc = RelationGetDescr(crel);
+ HeapTuple ctup;
+ Form_pg_class pgcform;
+ int replaces[3] = {0};
+ Datum values[3] = {0};
+ bool nulls[3] = {0};
+ int nreplaces = 0;
+
+ ctup = SearchSysCache1(RELOID, ObjectIdGetDatum(reloid));
+ if (!HeapTupleIsValid(ctup))
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_OBJECT_IN_USE),
+ errmsg("pg_class entry for relid %u not found", reloid)));
+ table_close(crel, RowExclusiveLock);
+ return false;
+ }
+ pgcform = (Form_pg_class) GETSTRUCT(ctup);
+
+ if (update_relpages && relpages != pgcform->relpages)
+ {
+ replaces[nreplaces] = Anum_pg_class_relpages;
+ values[nreplaces] = Int32GetDatum(relpages);
+ nreplaces++;
+ }
+
+ if (update_reltuples && reltuples != pgcform->reltuples)
+ {
+ replaces[nreplaces] = Anum_pg_class_reltuples;
+ values[nreplaces] = Float4GetDatum(reltuples);
+ nreplaces++;
+ }
+
+ if (update_relallvisible && relallvisible != pgcform->relallvisible)
+ {
+ replaces[nreplaces] = Anum_pg_class_relallvisible;
+ values[nreplaces] = Int32GetDatum(relallvisible);
+ nreplaces++;
+ }
+
+ if (nreplaces > 0)
+ {
+ HeapTuple newtup;
+
+ newtup = heap_modify_tuple_by_cols(ctup, tupdesc, nreplaces,
+ replaces, values, nulls);
+ CatalogTupleUpdate(crel, &newtup->t_self, newtup);
+ heap_freetuple(newtup);
+ }
- newtup = heap_modify_tuple_by_cols(ctup, tupdesc, ncols, replaces, values,
- nulls);
- CatalogTupleUpdate(crel, &newtup->t_self, newtup);
- heap_freetuple(newtup);
+ ReleaseSysCache(ctup);
}
/* release the lock, consistent with vac_update_relstats() */
@@ -188,7 +244,7 @@ relation_statistics_update(FunctionCallInfo fcinfo, int elevel)
Datum
pg_set_relation_stats(PG_FUNCTION_ARGS)
{
- relation_statistics_update(fcinfo, ERROR);
+ relation_statistics_update(fcinfo, ERROR, false);
PG_RETURN_VOID();
}
@@ -212,7 +268,7 @@ pg_clear_relation_stats(PG_FUNCTION_ARGS)
newfcinfo->args[3].value = DEFAULT_RELALLVISIBLE;
newfcinfo->args[3].isnull = false;
- relation_statistics_update(newfcinfo, ERROR);
+ relation_statistics_update(newfcinfo, ERROR, false);
PG_RETURN_VOID();
}
@@ -230,7 +286,7 @@ pg_restore_relation_stats(PG_FUNCTION_ARGS)
relarginfo, WARNING))
result = false;
- if (!relation_statistics_update(positional_fcinfo, WARNING))
+ if (!relation_statistics_update(positional_fcinfo, WARNING, true))
result = false;
PG_RETURN_BOOL(result);
diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out
index aab862c97c7..fb50da1cd83 100644
--- a/src/test/regress/expected/stats_import.out
+++ b/src/test/regress/expected/stats_import.out
@@ -105,6 +105,47 @@ WHERE oid = 'stats_import.test'::regclass;
18 | 401 | 5
(1 row)
+-- test MVCC behavior: changes do not persist after abort (in contrast
+-- to pg_restore_relation_stats(), which uses in-place updates).
+BEGIN;
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test'::regclass,
+ relpages => NULL::integer,
+ reltuples => 4000.0::real,
+ relallvisible => 4::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
+ABORT;
+SELECT relpages, reltuples, relallvisible
+FROM pg_class
+WHERE oid = 'stats_import.test'::regclass;
+ relpages | reltuples | relallvisible
+----------+-----------+---------------
+ 18 | 401 | 5
+(1 row)
+
+BEGIN;
+SELECT
+ pg_catalog.pg_clear_relation_stats(
+ 'stats_import.test'::regclass);
+ pg_clear_relation_stats
+-------------------------
+
+(1 row)
+
+ABORT;
+SELECT relpages, reltuples, relallvisible
+FROM pg_class
+WHERE oid = 'stats_import.test'::regclass;
+ relpages | reltuples | relallvisible
+----------+-----------+---------------
+ 18 | 401 | 5
+(1 row)
+
-- clear
SELECT
pg_catalog.pg_clear_relation_stats(
@@ -705,6 +746,25 @@ WHERE oid = 'stats_import.test'::regclass;
(1 row)
-- ok: just relpages
+SELECT pg_restore_relation_stats(
+ 'relation', 'stats_import.test'::regclass,
+ 'version', 150000::integer,
+ 'relpages', '15'::integer);
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
+SELECT relpages, reltuples, relallvisible
+FROM pg_class
+WHERE oid = 'stats_import.test'::regclass;
+ relpages | reltuples | relallvisible
+----------+-----------+---------------
+ 15 | 400 | 4
+(1 row)
+
+-- test non-MVCC behavior: new value should persist after abort
+BEGIN;
SELECT pg_restore_relation_stats(
'relation', 'stats_import.test'::regclass,
'version', 150000::integer,
@@ -714,6 +774,7 @@ SELECT pg_restore_relation_stats(
t
(1 row)
+ABORT;
SELECT relpages, reltuples, relallvisible
FROM pg_class
WHERE oid = 'stats_import.test'::regclass;
diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql
index 31455b58c1d..d3058bf8f6b 100644
--- a/src/test/regress/sql/stats_import.sql
+++ b/src/test/regress/sql/stats_import.sql
@@ -76,6 +76,31 @@ SELECT relpages, reltuples, relallvisible
FROM pg_class
WHERE oid = 'stats_import.test'::regclass;
+-- test MVCC behavior: changes do not persist after abort (in contrast
+-- to pg_restore_relation_stats(), which uses in-place updates).
+BEGIN;
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test'::regclass,
+ relpages => NULL::integer,
+ reltuples => 4000.0::real,
+ relallvisible => 4::integer);
+ABORT;
+
+SELECT relpages, reltuples, relallvisible
+FROM pg_class
+WHERE oid = 'stats_import.test'::regclass;
+
+BEGIN;
+SELECT
+ pg_catalog.pg_clear_relation_stats(
+ 'stats_import.test'::regclass);
+ABORT;
+
+SELECT relpages, reltuples, relallvisible
+FROM pg_class
+WHERE oid = 'stats_import.test'::regclass;
+
-- clear
SELECT
pg_catalog.pg_clear_relation_stats(
@@ -565,10 +590,22 @@ FROM pg_class
WHERE oid = 'stats_import.test'::regclass;
-- ok: just relpages
+SELECT pg_restore_relation_stats(
+ 'relation', 'stats_import.test'::regclass,
+ 'version', 150000::integer,
+ 'relpages', '15'::integer);
+
+SELECT relpages, reltuples, relallvisible
+FROM pg_class
+WHERE oid = 'stats_import.test'::regclass;
+
+-- test non-MVCC behavior: new value should persist after abort
+BEGIN;
SELECT pg_restore_relation_stats(
'relation', 'stats_import.test'::regclass,
'version', 150000::integer,
'relpages', '16'::integer);
+ABORT;
SELECT relpages, reltuples, relallvisible
FROM pg_class
--
2.34.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2024-12-07 19:27 ` Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2024-12-07 19:27 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Tue, 2024-11-26 at 17:11 -0500, Corey Huinker wrote:
>
> > * Why does binary upgrade cause statistics to be dumped? Can you
> > just
> > make pg_upgrade specify the appropriate set of flags?
> >
>
>
> That decision goes back a ways, I tried to dig in the archives last
> night but I was getting a Server Error on postgresql.org.
I suggest that pg_upgrade be changed to pass --no-data to pg_dump,
rather than --schema-only.
That way, you don't need to create a special case for the pg_dump
default that depends on whether it's a binary upgrade or not.
If wanted, there could also be a new option to pg_upgrade to specify --
with-statistics (default, passes --no-data to pg_dump) or --no-
statistics (passes --schema-only to pg_dump). But that option is
probably not necessary; everyone upgrading probably wants the stats.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2024-12-07 19:56 ` Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2024-12-07 19:56 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Sat, Dec 7, 2024 at 2:27 PM Jeff Davis <[email protected]> wrote:
> On Tue, 2024-11-26 at 17:11 -0500, Corey Huinker wrote:
> >
> > > * Why does binary upgrade cause statistics to be dumped? Can you
> > > just
> > > make pg_upgrade specify the appropriate set of flags?
> > >
> >
> >
> > That decision goes back a ways, I tried to dig in the archives last
> > night but I was getting a Server Error on postgresql.org.
>
> I suggest that pg_upgrade be changed to pass --no-data to pg_dump,
> rather than --schema-only.
>
> That way, you don't need to create a special case for the pg_dump
> default that depends on whether it's a binary upgrade or not.
>
+1
> If wanted, there could also be a new option to pg_upgrade to specify --
> with-statistics (default, passes --no-data to pg_dump) or --no-
> statistics (passes --schema-only to pg_dump). But that option is
> probably not necessary; everyone upgrading probably wants the stats.
>
This makes sense, though perhaps instead of --schema-only perhaps we should
pass both --no-statistics and --no-data. I don't envision a fourth option
to the new data/schema/stats triumvirate, but --no-statistics shouldn't
have a bearing on that future fourth option.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2024-12-07 21:27 ` Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2024-12-07 21:27 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Sat, 2024-12-07 at 14:56 -0500, Corey Huinker wrote:
> This makes sense, though perhaps instead of --schema-only perhaps we
> should pass both --no-statistics and --no-data. I don't envision a
> fourth option to the new data/schema/stats triumvirate, but --no-
> statistics shouldn't have a bearing on that future fourth option.
+1, assuming such an option is wanted at all. I suppose it should be
there for the unlikely (and hopefully impossible) case that statistics
are causing a problem during upgrade.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2024-12-12 03:49 ` Corey Huinker <[email protected]>
2024-12-13 05:22 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
0 siblings, 2 replies; 81+ messages in thread
From: Corey Huinker @ 2024-12-12 03:49 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
> +1, assuming such an option is wanted at all. I suppose it should be
> there for the unlikely (and hopefully impossible) case that statistics
> are causing a problem during upgrade.
>
Here you go, rebased and re-ordered:
0001-0004 are the pg_dump/pg_upgrade related patches.
0005 is an optimization to the attribute stats update
0006-0011 is the still-up-for-debate vacuumdb changes.
The patch for handling the as-yet-theoretical change to default relpages
for partitioned tables got messy in the rebase, so I decided to just leave
it out for now, as the change to relpages looks increasingly unlikely.
Attachments:
[text/x-patch] v34-0002-Add-no-data-option.patch (5.4K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/3-v34-0002-Add-no-data-option.patch)
download | inline diff:
From c10e1c1f87bc8039dfae38077d6b37446bcf408d Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 14 Nov 2024 04:58:17 -0500
Subject: [PATCH v34 02/11] Add --no-data option.
This option is useful for situations where someone wishes to test
query plans from a production database without copying production data.
---
src/bin/pg_dump/pg_backup.h | 2 ++
src/bin/pg_dump/pg_backup_archiver.c | 2 ++
src/bin/pg_dump/pg_dump.c | 7 ++++++-
src/bin/pg_dump/pg_restore.c | 5 ++++-
4 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 8fbb39d399..241855d017 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,6 +110,7 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
@@ -185,6 +186,7 @@ typedef struct _dumpOptions
int no_publications;
int no_subscriptions;
int no_statistics;
+ int no_data;
int no_toast_compression;
int no_unlogged_table_data;
int serializable_deferrable;
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 185d7fbb7e..41001e64ac 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -186,6 +186,8 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 8f85b04c33..c8a0b4afdf 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -494,6 +494,7 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
{"no-statistics", no_argument, &dopt.no_statistics, 1},
@@ -796,6 +797,9 @@ main(int argc, char **argv)
if (data_only && statistics_only)
pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+
if (statistics_only && dopt.no_statistics)
pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
@@ -812,7 +816,7 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpData = data_only || (!schema_only && !statistics_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
if (statistics_only)
@@ -1247,6 +1251,7 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
printf(_(" --no-statistics do not dump statistics\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 355f0439da..31c3cd32de 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -71,6 +71,7 @@ main(int argc, char **argv)
static int outputNoTableAm = 0;
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
+ static int no_data = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
@@ -132,6 +133,7 @@ main(int argc, char **argv)
{"no-subscriptions", no_argument, &no_subscriptions, 1},
{"filter", required_argument, NULL, 4},
{"no-statistics", no_argument, &no_statistics, 1},
+ {"no-data", no_argument, &no_data, 1},
{NULL, 0, NULL, 0}
};
@@ -375,7 +377,7 @@ main(int argc, char **argv)
/* set derivative flags */
opts->dumpSchema = (!data_only && !statistics_only);
- opts->dumpData = (!schema_only && !statistics_only);
+ opts->dumpData = (!no_data && !schema_only && !statistics_only);
opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
@@ -389,6 +391,7 @@ main(int argc, char **argv)
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
--
2.47.1
[text/x-patch] v34-0003-Change-pg_upgrade-s-invocation-of-pg_dump-to-use.patch (1.9K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/4-v34-0003-Change-pg_upgrade-s-invocation-of-pg_dump-to-use.patch)
download | inline diff:
From 0a648ab51f0cefa8bc74897a24d3310918133d7a Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 11 Dec 2024 14:28:54 -0500
Subject: [PATCH v34 03/11] Change pg_upgrade's invocation of pg_dump to use
--no-data
---
src/bin/pg_dump/pg_dump.c | 13 +------------
src/bin/pg_upgrade/dump.c | 2 +-
2 files changed, 2 insertions(+), 13 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index c8a0b4afdf..e626243f3a 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -818,18 +818,7 @@ main(int argc, char **argv)
/* set derivative flags */
dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
-
- if (statistics_only)
- /* stats are only thing wanted */
- dopt.dumpStatistics = true;
- else if (dopt.no_statistics)
- /* stats specifically excluded */
- dopt.dumpStatistics = false;
- else if (dopt.binary_upgrade)
- /* binary upgrade and not specifically excluded */
- dopt.dumpStatistics = true;
- else
- dopt.dumpStatistics = (!data_only && !schema_only);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8345f55be8..ef2f14a79b 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -52,7 +52,7 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
--
2.47.1
[text/x-patch] v34-0005-Consolidate-attribute-syscache-lookups-into-one-.patch (4.4K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/5-v34-0005-Consolidate-attribute-syscache-lookups-into-one-.patch)
download | inline diff:
From b0f1797617b8c33ecad93c83089b1cee27b1072f Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 14 Nov 2024 03:53:33 -0500
Subject: [PATCH v34 05/11] Consolidate attribute syscache lookups into one
call by name.
Previously we were doing one lookup by attname and one lookup by attnum,
which seems wasteful.
---
src/backend/statistics/attribute_stats.c | 55 +++++++++++-------------
1 file changed, 24 insertions(+), 31 deletions(-)
diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c
index b97ba7b0c0..6393783f8e 100644
--- a/src/backend/statistics/attribute_stats.c
+++ b/src/backend/statistics/attribute_stats.c
@@ -78,8 +78,8 @@ static struct StatsArgInfo attarginfo[] =
static bool attribute_statistics_update(FunctionCallInfo fcinfo, int elevel);
static Node *get_attr_expr(Relation rel, int attnum);
-static void get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
- Oid *atttypid, int32 *atttypmod,
+static void get_attr_stat_type(Oid reloid, Name attname, int elevel,
+ AttrNumber *attnum, Oid *atttypid, int32 *atttypmod,
char *atttyptype, Oid *atttypcoll,
Oid *eq_opr, Oid *lt_opr);
static bool get_elem_stat_type(Oid atttypid, char atttyptype, int elevel,
@@ -166,23 +166,16 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel)
stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG);
attname = PG_GETARG_NAME(ATTNAME_ARG);
- attnum = get_attnum(reloid, NameStr(*attname));
-
- if (attnum < 0)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("cannot modify statistics on system column \"%s\"",
- NameStr(*attname))));
-
- if (attnum == InvalidAttrNumber)
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("column \"%s\" of relation \"%s\" does not exist",
- NameStr(*attname), get_rel_name(reloid))));
stats_check_required_arg(fcinfo, attarginfo, INHERITED_ARG);
inherited = PG_GETARG_BOOL(INHERITED_ARG);
+ /* derive type information from attribute */
+ get_attr_stat_type(reloid, attname, elevel,
+ &attnum, &atttypid, &atttypmod,
+ &atttyptype, &atttypcoll,
+ &eq_opr, <_opr);
+
/*
* Check argument sanity. If some arguments are unusable, emit at elevel
* and set the corresponding argument to NULL in fcinfo.
@@ -232,12 +225,6 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel)
result = false;
}
- /* derive information from attribute */
- get_attr_stat_type(reloid, attnum, elevel,
- &atttypid, &atttypmod,
- &atttyptype, &atttypcoll,
- &eq_opr, <_opr);
-
/* if needed, derive element type */
if (do_mcelem || do_dechist)
{
@@ -503,8 +490,8 @@ get_attr_expr(Relation rel, int attnum)
* Derive type information from the attribute.
*/
static void
-get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
- Oid *atttypid, int32 *atttypmod,
+get_attr_stat_type(Oid reloid, Name attname, int elevel,
+ AttrNumber *attnum, Oid *atttypid, int32 *atttypmod,
char *atttyptype, Oid *atttypcoll,
Oid *eq_opr, Oid *lt_opr)
{
@@ -514,24 +501,30 @@ get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
Node *expr;
TypeCacheEntry *typcache;
- atup = SearchSysCache2(ATTNUM, ObjectIdGetDatum(reloid),
- Int16GetDatum(attnum));
+ atup = SearchSysCacheAttName(reloid, NameStr(*attname));
- /* Attribute not found */
+ /* Attribute not found or is dropped */
if (!HeapTupleIsValid(atup))
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("attribute %d of relation \"%s\" does not exist",
- attnum, RelationGetRelationName(rel))));
+ errmsg("column \"%s\" of relation \"%s\" does not exist",
+ NameStr(*attname), get_rel_name(reloid))));
attr = (Form_pg_attribute) GETSTRUCT(atup);
- if (attr->attisdropped)
+ if (attr->attnum < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot modify statistics on system column \"%s\"",
+ NameStr(*attname))));
+
+ if (attr->attnum == InvalidAttrNumber)
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("attribute %d of relation \"%s\" does not exist",
- attnum, RelationGetRelationName(rel))));
+ errmsg("column \"%s\" of relation \"%s\" does not exist",
+ NameStr(*attname), get_rel_name(reloid))));
+ *attnum = attr->attnum;
expr = get_attr_expr(rel, attr->attnum);
/*
--
2.47.1
[text/x-patch] v34-0004-Add-statistics-flags-to-pg_upgrade.patch (3.8K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/6-v34-0004-Add-statistics-flags-to-pg_upgrade.patch)
download | inline diff:
From e9dd923a80b2486c24224acc654c47da926d6fe6 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 11 Dec 2024 22:38:00 -0500
Subject: [PATCH v34 04/11] Add statistics flags to pg_upgrade
---
src/bin/pg_upgrade/dump.c | 6 ++++--
src/bin/pg_upgrade/option.c | 12 ++++++++++++
src/bin/pg_upgrade/pg_upgrade.h | 1 +
3 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index ef2f14a79b..954e7bacf4 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --no-data --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 6f41d63eed..76a0bb2991 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -307,6 +317,8 @@ usage(void)
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statisttics import statistics from old cluster (default)\n"));
+ printf(_(" --no-statisttics do not import statistics from old cluster\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 53f693c2d4..371f9a2d06 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
--
2.47.1
[text/x-patch] v34-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch (34.6K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/7-v34-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From d85115b777fe64c0f81a08d2b3abedc8bf9b3f65 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v34 01/11] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute. These
statements will restore the statistics of the current system onto the
destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, staistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index and Materialized View
statistics are dumped in the post-data section.
---
src/bin/pg_dump/pg_backup.h | 4 +
src/bin/pg_dump/pg_backup_archiver.c | 5 +
src/bin/pg_dump/pg_dump.c | 375 ++++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 8 +
src/bin/pg_dump/pg_dump_sort.c | 5 +
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 18 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
doc/src/sgml/ref/pg_dump.sgml | 36 ++-
doc/src/sgml/ref/pg_restore.sgml | 31 ++-
10 files changed, 489 insertions(+), 16 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b2..8fbb39d399 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -113,6 +113,7 @@ typedef struct _restoreOptions
int no_publications; /* Skip publication entries */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +161,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -182,6 +184,7 @@ typedef struct _dumpOptions
int no_security_labels;
int no_publications;
int no_subscriptions;
+ int no_statistics;
int no_toast_compression;
int no_unlogged_table_data;
int serializable_deferrable;
@@ -208,6 +211,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844..185d7fbb7e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -2962,6 +2962,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's a stats dump, maybe ignore it */
+ if (ropt->no_statistics && strcmp(te->desc, "STATISTICS") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +2995,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 89276524ae..8f85b04c33 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -430,6 +430,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -466,6 +467,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -494,6 +496,7 @@ main(int argc, char **argv)
{"no-comments", no_argument, &dopt.no_comments, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -539,7 +542,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:PRsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -613,6 +616,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -784,6 +791,13 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -798,8 +812,20 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
+
+ if (statistics_only)
+ /* stats are only thing wanted */
+ dopt.dumpStatistics = true;
+ else if (dopt.no_statistics)
+ /* stats specifically excluded */
+ dopt.dumpStatistics = false;
+ else if (dopt.binary_upgrade)
+ /* binary upgrade and not specifically excluded */
+ dopt.dumpStatistics = true;
+ else
+ dopt.dumpStatistics = (!data_only && !schema_only);
/*
* --inserts are already implied above if --column-inserts or
@@ -1099,6 +1125,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1177,7 +1204,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1190,11 +1217,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1221,6 +1249,7 @@ help(const char *progname)
printf(_(" --no-comments do not dump comment commands\n"));
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6777,6 +6806,42 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7154,6 +7219,7 @@ getTables(Archive *fout, int *numTables)
/* Tables have data */
tblinfo[i].dobj.components |= DUMP_COMPONENT_DATA;
+ tblinfo[i].dobj.components |= DUMP_COMPONENT_STATISTICS;
/* Mark whether table has an ACL */
if (!PQgetisnull(res, i, i_relacl))
@@ -7202,6 +7268,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7646,11 +7714,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7673,7 +7744,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7707,6 +7785,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10294,6 +10374,276 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * get statistics dump section, which depends on the parent object type
+ *
+ * objects created in SECTION_PRE_DATA have stats in SECTION_DATA
+ * objects created in SECTION_POST_DATA have stats in SECTION_POST_DATA
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+
+ if ((rsinfo->relkind == RELKIND_MATVIEW) ||
+ (rsinfo->relkind == RELKIND_INDEX) ||
+ (rsinfo->relkind == RELKIND_PARTITIONED_INDEX))
+ return SECTION_POST_DATA;
+
+ return SECTION_DATA;
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = dobj->dependencies,
+ .nDeps = dobj->nDeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10742,6 +11092,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -17172,6 +17525,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo)
free(indstatvalsarray);
}
+ /* Comments and stats share same .dep */
+
/* Dump Index Comments */
if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT)
dumpComment(fout, "INDEX", qindxname,
@@ -18959,6 +19314,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 2e55a0e3bb..def5f292e6 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -84,6 +84,7 @@ typedef enum
DO_PUBLICATION_TABLE_IN_SCHEMA,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
+ DO_REL_STATS,
} DumpableObjectType;
/*
@@ -101,6 +102,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -421,6 +423,12 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'v', 'c', etc */
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index 3c8f2eb808..989d20aa27 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -1500,6 +1500,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 9a04e51c81..62e2766c09 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 88ae39d938..355f0439da 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,7 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -74,6 +75,7 @@ main(int argc, char **argv)
static int no_publications = 0;
static int no_security_labels = 0;
static int no_subscriptions = 0;
+ static int no_statistics = 0;
static int strict_names = 0;
bool data_only = false;
bool schema_only = false;
@@ -108,6 +110,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'P'},
/*
* the following options don't have an equivalent short option letter
@@ -128,6 +131,7 @@ main(int argc, char **argv)
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
{"filter", required_argument, NULL, 4},
+ {"no-statistics", no_argument, &no_statistics, 1},
{NULL, 0, NULL, 0}
};
@@ -271,6 +275,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +351,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +374,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpSchema = (!data_only && !statistics_only);
+ opts->dumpData = (!schema_only && !statistics_only);
+ opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +388,7 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index b9d13a0e1d..76ebf15f55 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d66e901f51..aa4785c612 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,8 +141,9 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or
+ <option>--statistics-only</option> is specified. The <option>-b</option>
switch is therefore only useful to add large objects to dumps
where a specific schema or table has been requested. Note that
large objects are considered data and therefore will be included when
@@ -516,10 +517,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +654,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -833,7 +847,8 @@ PostgreSQL documentation
though you do not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1098,6 +1113,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719..ff1441a243 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,20 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema or data.
+ </para>
+ <para>
+ (Do not confuse this with the <option>--schema</option> option, which
+ uses the word <quote>schema</quote> in a different meaning.)
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -723,6 +738,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
--
2.47.1
[text/x-patch] v34-0006-split-out-check_conn_options.patch (4.5K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/8-v34-0006-split-out-check_conn_options.patch)
download | inline diff:
From 1108589023ac2789a5b9dd44fb35663e4af7e392 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:20:07 -0500
Subject: [PATCH v34 06/11] split out check_conn_options
---
src/bin/scripts/vacuumdb.c | 103 ++++++++++++++++++++-----------------
1 file changed, 57 insertions(+), 46 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index d07ab7d67e..7b97a9428a 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -10,6 +10,7 @@
*-------------------------------------------------------------------------
*/
+#include "libpq-fe.h"
#include "postgres_fe.h"
#include <limits.h>
@@ -459,55 +460,11 @@ escape_quotes(const char *src)
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
+ * Check connection options for compatibility with the connected database.
*/
static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
- PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
- SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
- bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
-
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
-
- conn = connectDatabase(cparams, progname, echo, false, true);
-
if (vacopts->disable_page_skipping && PQserverVersion(conn) < 90600)
{
PQfinish(conn);
@@ -575,6 +532,60 @@ vacuum_one_database(ConnParams *cparams,
/* skip_database_stats is used automatically if server supports it */
vacopts->skip_database_stats = (PQserverVersion(conn) >= 160000);
+}
+
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PQExpBufferData buf;
+ PQExpBufferData catalog_query;
+ PGresult *res;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ SimpleStringList dbtables = {NULL, NULL};
+ int i;
+ int ntups;
+ bool failed = false;
+ bool objects_listed = false;
+ const char *initcmd;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
if (!quiet)
{
--
2.47.1
[text/x-patch] v34-0008-split-out-generate_catalog_list.patch (7.1K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/9-v34-0008-split-out-generate_catalog_list.patch)
download | inline diff:
From 29f721142c9b7bb10f071db1461373aaec5586ed Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 16:15:16 -0500
Subject: [PATCH v34 08/11] split out generate_catalog_list
---
src/bin/scripts/vacuumdb.c | 176 +++++++++++++++++++++----------------
1 file changed, 99 insertions(+), 77 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index e9946f79b2..36f4796db0 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -560,67 +560,38 @@ print_processing_notice(PGconn *conn, int stage, const char *progname, bool quie
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
- */
-static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+ * Prepare the list of tables to process by querying the catalogs.
+ *
+ * Since we execute the constructed query with the default search_path
+ * (which could be unsafe), everything in this query MUST be fully
+ * qualified.
+ *
+ * First, build a WITH clause for the catalog query if any tables were
+ * specified, with a set of values made of relation names and their
+ * optional set of columns. This is used to match any provided column
+ * lists with the generated qualified identifiers and to filter for the
+ * tables provided via --table. If a listed table does not exist, the
+ * catalog query will fail.
+ */
+static SimpleStringList *
+generate_catalog_list(PGconn *conn,
+ vacuumingOptions *vacopts,
+ SimpleStringList *objects,
+ bool echo,
+ int *ntups)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
+ PQExpBufferData buf;
+ SimpleStringList *dbtables;
SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
+ PGresult *res;
+ int i;
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+ dbtables = palloc(sizeof(SimpleStringList));
+ dbtables->head = NULL;
+ dbtables->tail = NULL;
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- /*
- * Prepare the list of tables to process by querying the catalogs.
- *
- * Since we execute the constructed query with the default search_path
- * (which could be unsafe), everything in this query MUST be fully
- * qualified.
- *
- * First, build a WITH clause for the catalog query if any tables were
- * specified, with a set of values made of relation names and their
- * optional set of columns. This is used to match any provided column
- * lists with the generated qualified identifiers and to filter for the
- * tables provided via --table. If a listed table does not exist, the
- * catalog query will fail.
- */
initPQExpBuffer(&catalog_query);
for (cell = objects ? objects->head : NULL; cell; cell = cell->next)
{
@@ -771,40 +742,91 @@ vacuum_one_database(ConnParams *cparams,
appendPQExpBufferStr(&catalog_query, " ORDER BY c.relpages DESC;");
executeCommand(conn, "RESET search_path;", echo);
res = executeQuery(conn, catalog_query.data, echo);
+ *ntups = PQntuples(res);
termPQExpBuffer(&catalog_query);
PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo));
- /*
- * If no rows are returned, there are no matching tables, so we are done.
- */
- ntups = PQntuples(res);
- if (ntups == 0)
- {
- PQclear(res);
- PQfinish(conn);
- return;
- }
-
/*
* Build qualified identifiers for each table, including the column list
* if given.
*/
- initPQExpBuffer(&buf);
- for (i = 0; i < ntups; i++)
+ if (*ntups > 0)
{
- appendPQExpBufferStr(&buf,
- fmtQualifiedId(PQgetvalue(res, i, 1),
- PQgetvalue(res, i, 0)));
+ initPQExpBuffer(&buf);
+ for (i = 0; i < *ntups; i++)
+ {
+ appendPQExpBufferStr(&buf,
+ fmtQualifiedId(PQgetvalue(res, i, 1),
+ PQgetvalue(res, i, 0)));
- if (objects_listed && !PQgetisnull(res, i, 2))
- appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
+ if (objects_listed && !PQgetisnull(res, i, 2))
+ appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
- simple_string_list_append(&dbtables, buf.data);
- resetPQExpBuffer(&buf);
+ simple_string_list_append(dbtables, buf.data);
+ resetPQExpBuffer(&buf);
+ }
+ termPQExpBuffer(&buf);
}
- termPQExpBuffer(&buf);
PQclear(res);
+ return dbtables;
+}
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ int ntups;
+ bool failed = false;
+ const char *initcmd;
+ SimpleStringList *dbtables;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+
+ dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
+
+ /*
+ * If no rows are returned, there are no matching tables, so we are done.
+ */
+ if (ntups == 0)
+ {
+ PQfinish(conn);
+ return;
+ }
+
+
/*
* Ensure concurrentCons is sane. If there are more connections than
* vacuumable relations, we don't need to use them all.
@@ -837,7 +859,7 @@ vacuum_one_database(ConnParams *cparams,
initPQExpBuffer(&sql);
- cell = dbtables.head;
+ cell = dbtables->head;
do
{
const char *tabname = cell->val;
--
2.47.1
[text/x-patch] v34-0007-split-out-print_processing_notice.patch (2.4K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/10-v34-0007-split-out-print_processing_notice.patch)
download | inline diff:
From 0de5fd8cd0ebc8d6f0144ff4d4e7922b7a4cfe09 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:46:51 -0500
Subject: [PATCH v34 07/11] split out print_processing_notice
---
src/bin/scripts/vacuumdb.c | 41 +++++++++++++++++++++++---------------
1 file changed, 25 insertions(+), 16 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 7b97a9428a..e9946f79b2 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -535,6 +535,30 @@ check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
}
+/*
+ * print the processing notice for a database.
+ */
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet)
+{
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ if (!quiet)
+ {
+ if (stage != ANALYZE_NO_STAGE)
+ printf(_("%s: processing database \"%s\": %s\n"),
+ progname, PQdb(conn), _(stage_messages[stage]));
+ else
+ printf(_("%s: vacuuming database \"%s\"\n"),
+ progname, PQdb(conn));
+ fflush(stdout);
+ }
+}
+
/*
* vacuum_one_database
*
@@ -574,11 +598,6 @@ vacuum_one_database(ConnParams *cparams,
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
"RESET default_statistics_target;"
};
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
@@ -586,17 +605,7 @@ vacuum_one_database(ConnParams *cparams,
conn = connectDatabase(cparams, progname, echo, false, true);
check_conn_options(conn, vacopts);
-
- if (!quiet)
- {
- if (stage != ANALYZE_NO_STAGE)
- printf(_("%s: processing database \"%s\": %s\n"),
- progname, PQdb(conn), _(stage_messages[stage]));
- else
- printf(_("%s: vacuuming database \"%s\"\n"),
- progname, PQdb(conn));
- fflush(stdout);
- }
+ print_processing_notice(conn, stage, progname, quiet);
/*
* Prepare the list of tables to process by querying the catalogs.
--
2.47.1
[text/x-patch] v34-0010-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch (1.5K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/11-v34-0010-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch)
download | inline diff:
From 378de1ba4e7f0ddb7598bf7ff785a65996be21a4 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 7 Nov 2024 18:06:52 -0500
Subject: [PATCH v34 10/11] Add issues_sql_unlike, opposite of issues_sql_like
This is the same as issues_sql_like(), but the specified text is
prohibited from being in the output rather than required.
This became necessary to test that a command-line filter was in fact
filtering out certain output that a prior test required.
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 25 ++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 508e5e3917..1281071479 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2799,6 +2799,31 @@ sub issues_sql_like
}
=pod
+=item $node->issues_sql_unlike(cmd, prohibited_sql, test_name)
+
+Run a command on the node, then verify that $prohibited_sql does not appear
+in the server log file.
+
+=cut
+
+sub issues_sql_unlike
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+ my ($self, $cmd, $prohibited_sql, $test_name) = @_;
+
+ local %ENV = $self->_get_env();
+
+ my $log_location = -s $self->logfile;
+
+ my $result = PostgreSQL::Test::Utils::run_log($cmd);
+ ok($result, "@$cmd exit code 0");
+ my $log =
+ PostgreSQL::Test::Utils::slurp_file($self->logfile, $log_location);
+ unlike($log, $prohibited_sql, "$test_name: SQL found in server log");
+ return;
+}
+
=item $node->log_content()
--
2.47.1
[text/x-patch] v34-0009-preserve-catalog-lists-across-staged-runs.patch (6.7K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/12-v34-0009-preserve-catalog-lists-across-staged-runs.patch)
download | inline diff:
From 431cc9dc617e4ba314637b48c190e22736509b36 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 21:30:49 -0500
Subject: [PATCH v34 09/11] preserve catalog lists across staged runs
---
src/bin/scripts/vacuumdb.c | 113 ++++++++++++++++++++++++++-----------
1 file changed, 80 insertions(+), 33 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 36f4796db0..b13f3c4224 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -68,6 +68,9 @@ static void vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet);
static void vacuum_all_databases(ConnParams *cparams,
@@ -83,6 +86,14 @@ static void prepare_vacuum_command(PQExpBuffer sql, int serverVersion,
static void run_vacuum_command(PGconn *conn, const char *sql, bool echo,
const char *table);
+static void check_conn_options(PGconn *conn, vacuumingOptions *vacopts);
+
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet);
+
+static SimpleStringList * generate_catalog_list(PGconn *conn, vacuumingOptions *vacopts,
+ SimpleStringList *objects, bool echo, int *ntups);
+
static void help(const char *progname);
void check_objfilter(void);
@@ -386,6 +397,11 @@ main(int argc, char *argv[])
}
else
{
+ PGconn *conn;
+ int ntup;
+ SimpleStringList *found_objects;
+ int stage;
+
if (dbname == NULL)
{
if (getenv("PGDATABASE"))
@@ -397,25 +413,37 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ check_conn_options(conn, &vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
if (analyze_in_stages)
{
- int stage;
-
for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
{
- vacuum_one_database(&cparams, &vacopts,
- stage,
+ /* the last pass disconnected the conn */
+ if (stage > 0)
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
}
else
- vacuum_one_database(&cparams, &vacopts,
- ANALYZE_NO_STAGE,
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
@@ -791,16 +819,17 @@ vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet)
{
PQExpBufferData sql;
- PGconn *conn;
SimpleStringListCell *cell;
ParallelSlotArray *sa;
- int ntups;
bool failed = false;
const char *initcmd;
- SimpleStringList *dbtables;
+
const char *stage_commands[] = {
"SET default_statistics_target=1; SET vacuum_cost_delay=0;",
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
@@ -810,13 +839,6 @@ vacuum_one_database(ConnParams *cparams,
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
-
/*
* If no rows are returned, there are no matching tables, so we are done.
*/
@@ -928,7 +950,7 @@ finish:
}
/*
- * Vacuum/analyze all connectable databases.
+ * Vacuum/analyze all ccparams->override_dbname = PQgetvalue(result, i, 0);onnectable databases.
*
* In analyze-in-stages mode, we process all databases in one stage before
* moving on to the next stage. That ensure minimal stats are available
@@ -944,8 +966,13 @@ vacuum_all_databases(ConnParams *cparams,
{
PGconn *conn;
PGresult *result;
- int stage;
int i;
+ int stage;
+
+ SimpleStringList **found_objects;
+ int *num_tuples;
+
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
@@ -953,7 +980,33 @@ vacuum_all_databases(ConnParams *cparams,
echo);
PQfinish(conn);
- if (analyze_in_stages)
+ /*
+ * connect to each database, check validity of options,
+ * build the list of found objects per database,
+ * and run the first/only vacuum stage
+ */
+ found_objects = palloc(PQntuples(result) * sizeof(SimpleStringList *));
+ num_tuples = palloc(PQntuples(result) * sizeof (int));
+
+ for (i = 0; i < PQntuples(result); i++)
+ {
+ cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects[i] = generate_catalog_list(conn, vacopts, objects, echo, &num_tuples[i]);
+
+ vacuum_one_database(cparams, vacopts,
+ stage,
+ objects,
+ concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
+ progname, echo, quiet);
+ }
+
+ if (stage != ANALYZE_NO_STAGE)
{
/*
* When analyzing all databases in stages, we analyze them all in the
@@ -963,35 +1016,29 @@ vacuum_all_databases(ConnParams *cparams,
* This means we establish several times as many connections, but
* that's a secondary consideration.
*/
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
for (i = 0; i < PQntuples(result); i++)
{
cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(cparams, vacopts,
stage,
objects,
concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
progname, echo, quiet);
}
}
}
- else
- {
- for (i = 0; i < PQntuples(result); i++)
- {
- cparams->override_dbname = PQgetvalue(result, i, 0);
-
- vacuum_one_database(cparams, vacopts,
- ANALYZE_NO_STAGE,
- objects,
- concurrentCons,
- progname, echo, quiet);
- }
- }
PQclear(result);
+ pfree(found_objects);
+ pfree(num_tuples);
}
/*
--
2.47.1
[text/x-patch] v34-0011-Add-force-analyze-to-vacuumdb.patch (12.1K, ../../CADkLM=cWc+Zbjnpew9sg346DdRQJj8OSvZjg4fwvpLH10EViJQ@mail.gmail.com/13-v34-0011-Add-force-analyze-to-vacuumdb.patch)
download | inline diff:
From cf4e731db9ffaa4e89d7c5d14b32668529c8c89a Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Fri, 8 Nov 2024 12:27:50 -0500
Subject: [PATCH v34 11/11] Add --force-analyze to vacuumdb.
The vacuumdb options of --analyze-in-stages and --analyze-only are often
used after a restore from a dump or a pg_upgrade to quickly rebuild
stats on a databse.
However, now that stats are imported in most (but not all) cases,
running either of these commands will be at least partially redundant,
and will overwrite the stats that were just imported, which is a big
POLA violation.
We could add a new option such as --analyze-missing-in-stages, but that
wouldn't help the userbase that grown accustomed to running
--analyze-in-stages after an upgrade.
The least-bad option to handle the situation is to change the behavior
of --analyze-only and --analyze-in-stages to only analyze tables which
were missing stats before the vacuumdb started, but offer the
--force-analyze flag to restore the old behavior for those who truly
wanted it.
---
src/bin/scripts/t/100_vacuumdb.pl | 6 +-
src/bin/scripts/vacuumdb.c | 91 ++++++++++++++++++++++++-------
doc/src/sgml/ref/vacuumdb.sgml | 48 ++++++++++++++++
3 files changed, 125 insertions(+), 20 deletions(-)
diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl
index 1a2bcb4959..2d669391fe 100644
--- a/src/bin/scripts/t/100_vacuumdb.pl
+++ b/src/bin/scripts/t/100_vacuumdb.pl
@@ -127,9 +127,13 @@ $node->issues_sql_like(
qr/statement: VACUUM \(SKIP_DATABASE_STATS, ANALYZE\) public.vactable\(a, b\);/,
'vacuumdb --analyze with complete column list');
$node->issues_sql_like(
+ [ 'vacuumdb', '--analyze-only', '--force-analyze', '--table', 'vactable(b)', 'postgres' ],
+ qr/statement: ANALYZE public.vactable\(b\);/,
+ 'vacuumdb --analyze-only --force-analyze with partial column list');
+$node->issues_sql_unlike(
[ 'vacuumdb', '--analyze-only', '--table', 'vactable(b)', 'postgres' ],
qr/statement: ANALYZE public.vactable\(b\);/,
- 'vacuumdb --analyze-only with partial column list');
+ 'vacuumdb --analyze-only --force-analyze with partial column list skipping vacuumed tables');
$node->command_checks_all(
[ 'vacuumdb', '--analyze', '--table', 'vacview', 'postgres' ],
0,
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index b13f3c4224..1aa5c46af5 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -25,6 +25,7 @@
#include "fe_utils/query_utils.h"
#include "fe_utils/simple_list.h"
#include "fe_utils/string_utils.h"
+#include "pqexpbuffer.h"
/* vacuum options controlled by user flags */
@@ -47,6 +48,8 @@ typedef struct vacuumingOptions
bool process_main;
bool process_toast;
bool skip_database_stats;
+ bool analyze_in_stages;
+ bool force_analyze;
char *buffer_usage_limit;
} vacuumingOptions;
@@ -75,7 +78,6 @@ static void vacuum_one_database(ConnParams *cparams,
static void vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet);
@@ -140,6 +142,7 @@ main(int argc, char *argv[])
{"no-process-toast", no_argument, NULL, 11},
{"no-process-main", no_argument, NULL, 12},
{"buffer-usage-limit", required_argument, NULL, 13},
+ {"force-analyze", no_argument, NULL, 14},
{NULL, 0, NULL, 0}
};
@@ -156,7 +159,6 @@ main(int argc, char *argv[])
bool echo = false;
bool quiet = false;
vacuumingOptions vacopts;
- bool analyze_in_stages = false;
SimpleStringList objects = {NULL, NULL};
int concurrentCons = 1;
int tbl_count = 0;
@@ -170,6 +172,8 @@ main(int argc, char *argv[])
vacopts.do_truncate = true;
vacopts.process_main = true;
vacopts.process_toast = true;
+ vacopts.force_analyze = false;
+ vacopts.analyze_in_stages = false;
pg_logging_init(argv[0]);
progname = get_progname(argv[0]);
@@ -251,7 +255,7 @@ main(int argc, char *argv[])
maintenance_db = pg_strdup(optarg);
break;
case 3:
- analyze_in_stages = vacopts.analyze_only = true;
+ vacopts.analyze_in_stages = vacopts.analyze_only = true;
break;
case 4:
vacopts.disable_page_skipping = true;
@@ -287,6 +291,9 @@ main(int argc, char *argv[])
case 13:
vacopts.buffer_usage_limit = escape_quotes(optarg);
break;
+ case 14:
+ vacopts.force_analyze = true;
+ break;
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -372,6 +379,14 @@ main(int argc, char *argv[])
pg_fatal("cannot use the \"%s\" option with the \"%s\" option",
"buffer-usage-limit", "full");
+ /*
+ * --force-analyze is only valid when used with --analyze-only, -analyze,
+ * or --analyze-in-stages
+ */
+ if (vacopts.force_analyze && !vacopts.analyze_only && !vacopts.analyze_in_stages)
+ pg_fatal("can only use the \"%s\" option with \"%s\" or \"%s\"",
+ "--force-analyze", "-Z/--analyze-only", "--analyze-in-stages");
+
/* fill cparams except for dbname, which is set below */
cparams.pghost = host;
cparams.pgport = port;
@@ -390,7 +405,6 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
vacuum_all_databases(&cparams, &vacopts,
- analyze_in_stages,
&objects,
concurrentCons,
progname, echo, quiet);
@@ -413,20 +427,27 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts.analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectDatabase(&cparams, progname, echo, false, true);
check_conn_options(conn, &vacopts);
print_processing_notice(conn, stage, progname, quiet);
found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
+ vacuum_one_database(&cparams, &vacopts, stage,
+ &objects,
+ concurrentCons,
+ conn,
+ found_objects,
+ ntup,
+ progname, echo, quiet);
- if (analyze_in_stages)
+ if (stage != ANALYZE_NO_STAGE)
{
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
/* the last pass disconnected the conn */
- if (stage > 0)
- conn = connectDatabase(&cparams, progname, echo, false, true);
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(&cparams, &vacopts, stage,
&objects,
@@ -437,14 +458,6 @@ main(int argc, char *argv[])
progname, echo, quiet);
}
}
- else
- vacuum_one_database(&cparams, &vacopts, stage,
- &objects,
- concurrentCons,
- conn,
- found_objects,
- ntup,
- progname, echo, quiet);
}
exit(0);
@@ -763,6 +776,47 @@ generate_catalog_list(PGconn *conn,
vacopts->min_mxid_age);
}
+ /*
+ * If this query is for an analyze-only or analyze-in-stages, two
+ * upgrade-centric operations, and force-analyze is NOT set, then
+ * exclude any relations that already have their full compliment
+ * of attribute stats and extended stats.
+ *
+ *
+ */
+ if ((vacopts->analyze_only || vacopts->analyze_in_stages) &&
+ !vacopts->force_analyze)
+ {
+ /*
+ * The pg_class in question has no pg_statistic rows representing
+ * user-visible columns that lack a corresponding pg_statitic row.
+ * Currently no differentiation is made for whether the
+ * pg_statistic.stainherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_attribute AS a\n"
+ " WHERE a.attrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND a.attnum OPERATOR(pg_catalog.>) 0 AND NOT a.attisdropped\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic AS s\n"
+ " WHERE s.starelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND s.staattnum OPERATOR(pg_catalog.=) a.attnum))\n");
+
+ /*
+ * The pg_class entry has no pg_statistic_ext rows that lack a corresponding
+ * pg_statistic_ext_data row. Currently no differentiation is made for whether
+ * pg_statistic_exta_data.stxdinherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext AS e\n"
+ " WHERE e.stxrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext_data AS d\n"
+ " WHERE d.stxoid OPERATOR(pg_catalog.=) e.oid))\n");
+ }
+
/*
* Execute the catalog query. We use the default search_path for this
* query for consistency with table lookups done elsewhere by the user.
@@ -959,7 +1013,6 @@ finish:
static void
vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet)
@@ -972,7 +1025,7 @@ vacuum_all_databases(ConnParams *cparams,
SimpleStringList **found_objects;
int *num_tuples;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts->analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml
index 66fccb30a2..00ec927606 100644
--- a/doc/src/sgml/ref/vacuumdb.sgml
+++ b/doc/src/sgml/ref/vacuumdb.sgml
@@ -425,6 +425,12 @@ PostgreSQL documentation
<para>
Only calculate statistics for use by the optimizer (no vacuum).
</para>
+ <para>
+ By default, this operation excludes relations that already have
+ statistics generated. If the option <option>--force-analyze</option>
+ is also specified, then relations with existing stastistics are not
+ excluded.
+ </para>
</listitem>
</varlistentry>
@@ -439,6 +445,47 @@ PostgreSQL documentation
to produce usable statistics faster, and subsequent stages build the
full statistics.
</para>
+ <para>
+ This option is intended to be run after a <command>pg_upgrade</command>
+ to generate statistics for relations that have no stistatics or incomplete
+ statistics (such as those with extended statistics objects, which are not
+ imported on upgrade).
+ </para>
+ <para>
+ If the option <option>--force-analyze</option> is also specified, then
+ relations with existing stastistics are not excluded.
+ This option is only useful to analyze a database that currently has
+ no statistics or has wholly incorrect ones, such as if it is newly
+ populated from a restored dump.
+ Be aware that running with this option combinationin a database with
+ existing statistics may cause the query optimizer choices to become
+ transiently worse due to the low statistics targets of the early
+ stages.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--force-analyze</option></term>
+ <listitem>
+ <para>
+ This option can only be used if either <option>--analyze-only</option>
+ or <option>--analyze-in-stages</option> is specified. It modifies those
+ options to not filter out relations that already have statistics.
+ </para>
+ <para>
+
+ Only calculate statistics for use by the optimizer (no vacuum),
+ like <option>--analyze-only</option>. Run three
+ stages of analyze; the first stage uses the lowest possible statistics
+ target (see <xref linkend="guc-default-statistics-target"/>)
+ to produce usable statistics faster, and subsequent stages build the
+ full statistics.
+ </para>
+
+ <para>
+ This option was created
+ </para>
<para>
This option is only useful to analyze a database that currently has
@@ -452,6 +499,7 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+
<varlistentry>
<term><option>-?</option></term>
<term><option>--help</option></term>
--
2.47.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2024-12-13 05:22 ` Corey Huinker <[email protected]>
2024-12-20 05:23 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2024-12-13 05:22 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Per offline conversation with Jeff, adding a --no-schema to pg_dump option
both for completeness (we already have --no-data and --no-statistics), but
users who previously got the effect of --no-schema did so by specifying
--data-only, which suppresses statistics as well.
0001-0005 - changes to pg_dump/pg_upgrade
0006 - attribute stats optimization
0007-0012 - vacuumdb
On Wed, Dec 11, 2024 at 10:49 PM Corey Huinker <[email protected]>
wrote:
> +1, assuming such an option is wanted at all. I suppose it should be
>> there for the unlikely (and hopefully impossible) case that statistics
>> are causing a problem during upgrade.
>>
>
> Here you go, rebased and re-ordered:
>
> 0001-0004 are the pg_dump/pg_upgrade related patches.
> 0005 is an optimization to the attribute stats update
> 0006-0011 is the still-up-for-debate vacuumdb changes.
>
> The patch for handling the as-yet-theoretical change to default relpages
> for partitioned tables got messy in the rebase, so I decided to just leave
> it out for now, as the change to relpages looks increasingly unlikely.
>
>
>
Attachments:
[text/x-patch] v35-0004-Add-no-schema-option-to-pg_dump-etc.patch (6.6K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/3-v35-0004-Add-no-schema-option-to-pg_dump-etc.patch)
download | inline diff:
From 27674ebea83387e0d714239effac486a30d8d70d Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 12 Dec 2024 16:58:21 -0500
Subject: [PATCH v35 04/12] Add --no-schema option to pg_dump, etc.
Previously, users could use --data-only when they wanted to suppress
schema from a dump. However, that no longer makes sense now that the
data/schema binary has become the data/schema/statistics trinary.
---
src/bin/pg_dump/pg_backup.h | 2 ++
src/bin/pg_dump/pg_backup_archiver.c | 1 +
src/bin/pg_dump/pg_dump.c | 6 ++++--
src/bin/pg_dump/pg_restore.c | 6 +++++-
doc/src/sgml/ref/pg_dump.sgml | 9 +++++++++
doc/src/sgml/ref/pg_restore.sgml | 10 ++++++++++
6 files changed, 31 insertions(+), 3 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 241855d017..24295110cf 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -112,6 +112,7 @@ typedef struct _restoreOptions
int no_comments; /* Skip comments */
int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
int no_statistics; /* Skip statistics import */
@@ -187,6 +188,7 @@ typedef struct _dumpOptions
int no_subscriptions;
int no_statistics;
int no_data;
+ int no_schema;
int no_toast_compression;
int no_unlogged_table_data;
int serializable_deferrable;
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 41001e64ac..d62f419560 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -187,6 +187,7 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index e626243f3a..61f90d7f78 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -497,6 +497,7 @@ main(int argc, char **argv)
{"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
@@ -799,7 +800,8 @@ main(int argc, char **argv)
if (data_only && dopt.no_data)
pg_fatal("options -a/--data-only and --no-data cannot be used together");
-
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
if (statistics_only && dopt.no_statistics)
pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
@@ -817,7 +819,7 @@ main(int argc, char **argv)
/* set derivative flags */
dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
- dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 31c3cd32de..90db03f0d4 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -72,6 +72,7 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_data = 0;
+ static int no_schema = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
@@ -134,6 +135,7 @@ main(int argc, char **argv)
{"filter", required_argument, NULL, 4},
{"no-statistics", no_argument, &no_statistics, 1},
{"no-data", no_argument, &no_data, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{NULL, 0, NULL, 0}
};
@@ -376,7 +378,7 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only && !statistics_only);
+ opts->dumpSchema = (!no_schema && !data_only && !statistics_only);
opts->dumpData = (!no_data && !schema_only && !statistics_only);
opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
@@ -511,7 +513,9 @@ usage(const char *progname)
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index aa4785c612..889b8d4e18 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -1113,6 +1113,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-statistics</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index ff1441a243..cc9dbb4808 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -738,6 +738,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-statistics</option></term>
<listitem>
--
2.47.1
[text/x-patch] v35-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch (34.6K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/4-v35-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From a84983e60b37693b02362e1dacfda1bcca92dd91 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v35 01/12] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute. These
statements will restore the statistics of the current system onto the
destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, staistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index and Materialized View
statistics are dumped in the post-data section.
---
src/bin/pg_dump/pg_backup.h | 4 +
src/bin/pg_dump/pg_backup_archiver.c | 5 +
src/bin/pg_dump/pg_dump.c | 375 ++++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 8 +
src/bin/pg_dump/pg_dump_sort.c | 5 +
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 18 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
doc/src/sgml/ref/pg_dump.sgml | 36 ++-
doc/src/sgml/ref/pg_restore.sgml | 31 ++-
10 files changed, 489 insertions(+), 16 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b2..8fbb39d399 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -113,6 +113,7 @@ typedef struct _restoreOptions
int no_publications; /* Skip publication entries */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +161,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -182,6 +184,7 @@ typedef struct _dumpOptions
int no_security_labels;
int no_publications;
int no_subscriptions;
+ int no_statistics;
int no_toast_compression;
int no_unlogged_table_data;
int serializable_deferrable;
@@ -208,6 +211,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844..185d7fbb7e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -2962,6 +2962,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's a stats dump, maybe ignore it */
+ if (ropt->no_statistics && strcmp(te->desc, "STATISTICS") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +2995,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 89276524ae..8f85b04c33 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -430,6 +430,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -466,6 +467,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -494,6 +496,7 @@ main(int argc, char **argv)
{"no-comments", no_argument, &dopt.no_comments, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -539,7 +542,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:PRsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -613,6 +616,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -784,6 +791,13 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -798,8 +812,20 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
+
+ if (statistics_only)
+ /* stats are only thing wanted */
+ dopt.dumpStatistics = true;
+ else if (dopt.no_statistics)
+ /* stats specifically excluded */
+ dopt.dumpStatistics = false;
+ else if (dopt.binary_upgrade)
+ /* binary upgrade and not specifically excluded */
+ dopt.dumpStatistics = true;
+ else
+ dopt.dumpStatistics = (!data_only && !schema_only);
/*
* --inserts are already implied above if --column-inserts or
@@ -1099,6 +1125,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1177,7 +1204,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1190,11 +1217,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1221,6 +1249,7 @@ help(const char *progname)
printf(_(" --no-comments do not dump comment commands\n"));
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6777,6 +6806,42 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7154,6 +7219,7 @@ getTables(Archive *fout, int *numTables)
/* Tables have data */
tblinfo[i].dobj.components |= DUMP_COMPONENT_DATA;
+ tblinfo[i].dobj.components |= DUMP_COMPONENT_STATISTICS;
/* Mark whether table has an ACL */
if (!PQgetisnull(res, i, i_relacl))
@@ -7202,6 +7268,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7646,11 +7714,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7673,7 +7744,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7707,6 +7785,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10294,6 +10374,276 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * get statistics dump section, which depends on the parent object type
+ *
+ * objects created in SECTION_PRE_DATA have stats in SECTION_DATA
+ * objects created in SECTION_POST_DATA have stats in SECTION_POST_DATA
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+
+ if ((rsinfo->relkind == RELKIND_MATVIEW) ||
+ (rsinfo->relkind == RELKIND_INDEX) ||
+ (rsinfo->relkind == RELKIND_PARTITIONED_INDEX))
+ return SECTION_POST_DATA;
+
+ return SECTION_DATA;
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = dobj->dependencies,
+ .nDeps = dobj->nDeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10742,6 +11092,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -17172,6 +17525,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo)
free(indstatvalsarray);
}
+ /* Comments and stats share same .dep */
+
/* Dump Index Comments */
if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT)
dumpComment(fout, "INDEX", qindxname,
@@ -18959,6 +19314,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 2e55a0e3bb..def5f292e6 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -84,6 +84,7 @@ typedef enum
DO_PUBLICATION_TABLE_IN_SCHEMA,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
+ DO_REL_STATS,
} DumpableObjectType;
/*
@@ -101,6 +102,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -421,6 +423,12 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'v', 'c', etc */
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index 3c8f2eb808..989d20aa27 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -1500,6 +1500,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 9a04e51c81..62e2766c09 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 88ae39d938..355f0439da 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,7 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -74,6 +75,7 @@ main(int argc, char **argv)
static int no_publications = 0;
static int no_security_labels = 0;
static int no_subscriptions = 0;
+ static int no_statistics = 0;
static int strict_names = 0;
bool data_only = false;
bool schema_only = false;
@@ -108,6 +110,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'P'},
/*
* the following options don't have an equivalent short option letter
@@ -128,6 +131,7 @@ main(int argc, char **argv)
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
{"filter", required_argument, NULL, 4},
+ {"no-statistics", no_argument, &no_statistics, 1},
{NULL, 0, NULL, 0}
};
@@ -271,6 +275,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +351,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +374,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpSchema = (!data_only && !statistics_only);
+ opts->dumpData = (!schema_only && !statistics_only);
+ opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +388,7 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index b9d13a0e1d..76ebf15f55 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d66e901f51..aa4785c612 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,8 +141,9 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or
+ <option>--statistics-only</option> is specified. The <option>-b</option>
switch is therefore only useful to add large objects to dumps
where a specific schema or table has been requested. Note that
large objects are considered data and therefore will be included when
@@ -516,10 +517,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +654,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -833,7 +847,8 @@ PostgreSQL documentation
though you do not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1098,6 +1113,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719..ff1441a243 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,20 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema or data.
+ </para>
+ <para>
+ (Do not confuse this with the <option>--schema</option> option, which
+ uses the word <quote>schema</quote> in a different meaning.)
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -723,6 +738,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
--
2.47.1
[text/x-patch] v35-0005-Add-statistics-flags-to-pg_upgrade.patch (3.8K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/5-v35-0005-Add-statistics-flags-to-pg_upgrade.patch)
download | inline diff:
From ab4a753d50cd2fbea7e8cd3e8b1dfa37f4ac9508 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 11 Dec 2024 22:38:00 -0500
Subject: [PATCH v35 05/12] Add statistics flags to pg_upgrade
---
src/bin/pg_upgrade/dump.c | 6 ++++--
src/bin/pg_upgrade/option.c | 12 ++++++++++++
src/bin/pg_upgrade/pg_upgrade.h | 1 +
3 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index ef2f14a79b..954e7bacf4 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --no-data --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 6f41d63eed..76a0bb2991 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -307,6 +317,8 @@ usage(void)
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statisttics import statistics from old cluster (default)\n"));
+ printf(_(" --no-statisttics do not import statistics from old cluster\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 53f693c2d4..371f9a2d06 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
--
2.47.1
[text/x-patch] v35-0002-Add-no-data-option.patch (5.4K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/6-v35-0002-Add-no-data-option.patch)
download | inline diff:
From 436f30b7bce2cd6eb9904384ca0395a7abab6bfd Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 14 Nov 2024 04:58:17 -0500
Subject: [PATCH v35 02/12] Add --no-data option.
This option is useful for situations where someone wishes to test
query plans from a production database without copying production data.
---
src/bin/pg_dump/pg_backup.h | 2 ++
src/bin/pg_dump/pg_backup_archiver.c | 2 ++
src/bin/pg_dump/pg_dump.c | 7 ++++++-
src/bin/pg_dump/pg_restore.c | 5 ++++-
4 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 8fbb39d399..241855d017 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,6 +110,7 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
@@ -185,6 +186,7 @@ typedef struct _dumpOptions
int no_publications;
int no_subscriptions;
int no_statistics;
+ int no_data;
int no_toast_compression;
int no_unlogged_table_data;
int serializable_deferrable;
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 185d7fbb7e..41001e64ac 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -186,6 +186,8 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 8f85b04c33..c8a0b4afdf 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -494,6 +494,7 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
{"no-statistics", no_argument, &dopt.no_statistics, 1},
@@ -796,6 +797,9 @@ main(int argc, char **argv)
if (data_only && statistics_only)
pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+
if (statistics_only && dopt.no_statistics)
pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
@@ -812,7 +816,7 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpData = data_only || (!schema_only && !statistics_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
if (statistics_only)
@@ -1247,6 +1251,7 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
printf(_(" --no-statistics do not dump statistics\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 355f0439da..31c3cd32de 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -71,6 +71,7 @@ main(int argc, char **argv)
static int outputNoTableAm = 0;
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
+ static int no_data = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
@@ -132,6 +133,7 @@ main(int argc, char **argv)
{"no-subscriptions", no_argument, &no_subscriptions, 1},
{"filter", required_argument, NULL, 4},
{"no-statistics", no_argument, &no_statistics, 1},
+ {"no-data", no_argument, &no_data, 1},
{NULL, 0, NULL, 0}
};
@@ -375,7 +377,7 @@ main(int argc, char **argv)
/* set derivative flags */
opts->dumpSchema = (!data_only && !statistics_only);
- opts->dumpData = (!schema_only && !statistics_only);
+ opts->dumpData = (!no_data && !schema_only && !statistics_only);
opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
@@ -389,6 +391,7 @@ main(int argc, char **argv)
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
--
2.47.1
[text/x-patch] v35-0003-Change-pg_upgrade-s-invocation-of-pg_dump-to-use.patch (1.9K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/7-v35-0003-Change-pg_upgrade-s-invocation-of-pg_dump-to-use.patch)
download | inline diff:
From 6b18d68111bcf25e66d76e792a2ebdc4543094cb Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 11 Dec 2024 14:28:54 -0500
Subject: [PATCH v35 03/12] Change pg_upgrade's invocation of pg_dump to use
--no-data
---
src/bin/pg_dump/pg_dump.c | 13 +------------
src/bin/pg_upgrade/dump.c | 2 +-
2 files changed, 2 insertions(+), 13 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index c8a0b4afdf..e626243f3a 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -818,18 +818,7 @@ main(int argc, char **argv)
/* set derivative flags */
dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
dopt.dumpSchema = schema_only || (!data_only && !statistics_only);
-
- if (statistics_only)
- /* stats are only thing wanted */
- dopt.dumpStatistics = true;
- else if (dopt.no_statistics)
- /* stats specifically excluded */
- dopt.dumpStatistics = false;
- else if (dopt.binary_upgrade)
- /* binary upgrade and not specifically excluded */
- dopt.dumpStatistics = true;
- else
- dopt.dumpStatistics = (!data_only && !schema_only);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8345f55be8..ef2f14a79b 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -52,7 +52,7 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
--
2.47.1
[text/x-patch] v35-0007-split-out-check_conn_options.patch (4.5K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/8-v35-0007-split-out-check_conn_options.patch)
download | inline diff:
From 4702b7b21f8b306d66650bb4e8354de9a131522c Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:20:07 -0500
Subject: [PATCH v35 07/12] split out check_conn_options
---
src/bin/scripts/vacuumdb.c | 103 ++++++++++++++++++++-----------------
1 file changed, 57 insertions(+), 46 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index d07ab7d67e..7b97a9428a 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -10,6 +10,7 @@
*-------------------------------------------------------------------------
*/
+#include "libpq-fe.h"
#include "postgres_fe.h"
#include <limits.h>
@@ -459,55 +460,11 @@ escape_quotes(const char *src)
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
+ * Check connection options for compatibility with the connected database.
*/
static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
- PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
- SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
- bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
-
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
-
- conn = connectDatabase(cparams, progname, echo, false, true);
-
if (vacopts->disable_page_skipping && PQserverVersion(conn) < 90600)
{
PQfinish(conn);
@@ -575,6 +532,60 @@ vacuum_one_database(ConnParams *cparams,
/* skip_database_stats is used automatically if server supports it */
vacopts->skip_database_stats = (PQserverVersion(conn) >= 160000);
+}
+
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PQExpBufferData buf;
+ PQExpBufferData catalog_query;
+ PGresult *res;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ SimpleStringList dbtables = {NULL, NULL};
+ int i;
+ int ntups;
+ bool failed = false;
+ bool objects_listed = false;
+ const char *initcmd;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
if (!quiet)
{
--
2.47.1
[text/x-patch] v35-0006-Consolidate-attribute-syscache-lookups-into-one-.patch (4.4K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/9-v35-0006-Consolidate-attribute-syscache-lookups-into-one-.patch)
download | inline diff:
From 8dbae3174f3a6c8bb72beb577dc5c32ccb996e65 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 14 Nov 2024 03:53:33 -0500
Subject: [PATCH v35 06/12] Consolidate attribute syscache lookups into one
call by name.
Previously we were doing one lookup by attname and one lookup by attnum,
which seems wasteful.
---
src/backend/statistics/attribute_stats.c | 55 +++++++++++-------------
1 file changed, 24 insertions(+), 31 deletions(-)
diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c
index b97ba7b0c0..6393783f8e 100644
--- a/src/backend/statistics/attribute_stats.c
+++ b/src/backend/statistics/attribute_stats.c
@@ -78,8 +78,8 @@ static struct StatsArgInfo attarginfo[] =
static bool attribute_statistics_update(FunctionCallInfo fcinfo, int elevel);
static Node *get_attr_expr(Relation rel, int attnum);
-static void get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
- Oid *atttypid, int32 *atttypmod,
+static void get_attr_stat_type(Oid reloid, Name attname, int elevel,
+ AttrNumber *attnum, Oid *atttypid, int32 *atttypmod,
char *atttyptype, Oid *atttypcoll,
Oid *eq_opr, Oid *lt_opr);
static bool get_elem_stat_type(Oid atttypid, char atttyptype, int elevel,
@@ -166,23 +166,16 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel)
stats_check_required_arg(fcinfo, attarginfo, ATTNAME_ARG);
attname = PG_GETARG_NAME(ATTNAME_ARG);
- attnum = get_attnum(reloid, NameStr(*attname));
-
- if (attnum < 0)
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("cannot modify statistics on system column \"%s\"",
- NameStr(*attname))));
-
- if (attnum == InvalidAttrNumber)
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("column \"%s\" of relation \"%s\" does not exist",
- NameStr(*attname), get_rel_name(reloid))));
stats_check_required_arg(fcinfo, attarginfo, INHERITED_ARG);
inherited = PG_GETARG_BOOL(INHERITED_ARG);
+ /* derive type information from attribute */
+ get_attr_stat_type(reloid, attname, elevel,
+ &attnum, &atttypid, &atttypmod,
+ &atttyptype, &atttypcoll,
+ &eq_opr, <_opr);
+
/*
* Check argument sanity. If some arguments are unusable, emit at elevel
* and set the corresponding argument to NULL in fcinfo.
@@ -232,12 +225,6 @@ attribute_statistics_update(FunctionCallInfo fcinfo, int elevel)
result = false;
}
- /* derive information from attribute */
- get_attr_stat_type(reloid, attnum, elevel,
- &atttypid, &atttypmod,
- &atttyptype, &atttypcoll,
- &eq_opr, <_opr);
-
/* if needed, derive element type */
if (do_mcelem || do_dechist)
{
@@ -503,8 +490,8 @@ get_attr_expr(Relation rel, int attnum)
* Derive type information from the attribute.
*/
static void
-get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
- Oid *atttypid, int32 *atttypmod,
+get_attr_stat_type(Oid reloid, Name attname, int elevel,
+ AttrNumber *attnum, Oid *atttypid, int32 *atttypmod,
char *atttyptype, Oid *atttypcoll,
Oid *eq_opr, Oid *lt_opr)
{
@@ -514,24 +501,30 @@ get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
Node *expr;
TypeCacheEntry *typcache;
- atup = SearchSysCache2(ATTNUM, ObjectIdGetDatum(reloid),
- Int16GetDatum(attnum));
+ atup = SearchSysCacheAttName(reloid, NameStr(*attname));
- /* Attribute not found */
+ /* Attribute not found or is dropped */
if (!HeapTupleIsValid(atup))
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("attribute %d of relation \"%s\" does not exist",
- attnum, RelationGetRelationName(rel))));
+ errmsg("column \"%s\" of relation \"%s\" does not exist",
+ NameStr(*attname), get_rel_name(reloid))));
attr = (Form_pg_attribute) GETSTRUCT(atup);
- if (attr->attisdropped)
+ if (attr->attnum < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot modify statistics on system column \"%s\"",
+ NameStr(*attname))));
+
+ if (attr->attnum == InvalidAttrNumber)
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_COLUMN),
- errmsg("attribute %d of relation \"%s\" does not exist",
- attnum, RelationGetRelationName(rel))));
+ errmsg("column \"%s\" of relation \"%s\" does not exist",
+ NameStr(*attname), get_rel_name(reloid))));
+ *attnum = attr->attnum;
expr = get_attr_expr(rel, attr->attnum);
/*
--
2.47.1
[text/x-patch] v35-0008-split-out-print_processing_notice.patch (2.4K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/10-v35-0008-split-out-print_processing_notice.patch)
download | inline diff:
From 1e405e14378c9184bad4bde4e19f2dda962df168 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:46:51 -0500
Subject: [PATCH v35 08/12] split out print_processing_notice
---
src/bin/scripts/vacuumdb.c | 41 +++++++++++++++++++++++---------------
1 file changed, 25 insertions(+), 16 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 7b97a9428a..e9946f79b2 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -535,6 +535,30 @@ check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
}
+/*
+ * print the processing notice for a database.
+ */
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet)
+{
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ if (!quiet)
+ {
+ if (stage != ANALYZE_NO_STAGE)
+ printf(_("%s: processing database \"%s\": %s\n"),
+ progname, PQdb(conn), _(stage_messages[stage]));
+ else
+ printf(_("%s: vacuuming database \"%s\"\n"),
+ progname, PQdb(conn));
+ fflush(stdout);
+ }
+}
+
/*
* vacuum_one_database
*
@@ -574,11 +598,6 @@ vacuum_one_database(ConnParams *cparams,
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
"RESET default_statistics_target;"
};
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
@@ -586,17 +605,7 @@ vacuum_one_database(ConnParams *cparams,
conn = connectDatabase(cparams, progname, echo, false, true);
check_conn_options(conn, vacopts);
-
- if (!quiet)
- {
- if (stage != ANALYZE_NO_STAGE)
- printf(_("%s: processing database \"%s\": %s\n"),
- progname, PQdb(conn), _(stage_messages[stage]));
- else
- printf(_("%s: vacuuming database \"%s\"\n"),
- progname, PQdb(conn));
- fflush(stdout);
- }
+ print_processing_notice(conn, stage, progname, quiet);
/*
* Prepare the list of tables to process by querying the catalogs.
--
2.47.1
[text/x-patch] v35-0009-split-out-generate_catalog_list.patch (7.1K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/11-v35-0009-split-out-generate_catalog_list.patch)
download | inline diff:
From 10b12c50cf784741c763b3a0b6635e8d07b9c713 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 16:15:16 -0500
Subject: [PATCH v35 09/12] split out generate_catalog_list
---
src/bin/scripts/vacuumdb.c | 176 +++++++++++++++++++++----------------
1 file changed, 99 insertions(+), 77 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index e9946f79b2..36f4796db0 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -560,67 +560,38 @@ print_processing_notice(PGconn *conn, int stage, const char *progname, bool quie
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
- */
-static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+ * Prepare the list of tables to process by querying the catalogs.
+ *
+ * Since we execute the constructed query with the default search_path
+ * (which could be unsafe), everything in this query MUST be fully
+ * qualified.
+ *
+ * First, build a WITH clause for the catalog query if any tables were
+ * specified, with a set of values made of relation names and their
+ * optional set of columns. This is used to match any provided column
+ * lists with the generated qualified identifiers and to filter for the
+ * tables provided via --table. If a listed table does not exist, the
+ * catalog query will fail.
+ */
+static SimpleStringList *
+generate_catalog_list(PGconn *conn,
+ vacuumingOptions *vacopts,
+ SimpleStringList *objects,
+ bool echo,
+ int *ntups)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
+ PQExpBufferData buf;
+ SimpleStringList *dbtables;
SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
+ PGresult *res;
+ int i;
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+ dbtables = palloc(sizeof(SimpleStringList));
+ dbtables->head = NULL;
+ dbtables->tail = NULL;
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- /*
- * Prepare the list of tables to process by querying the catalogs.
- *
- * Since we execute the constructed query with the default search_path
- * (which could be unsafe), everything in this query MUST be fully
- * qualified.
- *
- * First, build a WITH clause for the catalog query if any tables were
- * specified, with a set of values made of relation names and their
- * optional set of columns. This is used to match any provided column
- * lists with the generated qualified identifiers and to filter for the
- * tables provided via --table. If a listed table does not exist, the
- * catalog query will fail.
- */
initPQExpBuffer(&catalog_query);
for (cell = objects ? objects->head : NULL; cell; cell = cell->next)
{
@@ -771,40 +742,91 @@ vacuum_one_database(ConnParams *cparams,
appendPQExpBufferStr(&catalog_query, " ORDER BY c.relpages DESC;");
executeCommand(conn, "RESET search_path;", echo);
res = executeQuery(conn, catalog_query.data, echo);
+ *ntups = PQntuples(res);
termPQExpBuffer(&catalog_query);
PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo));
- /*
- * If no rows are returned, there are no matching tables, so we are done.
- */
- ntups = PQntuples(res);
- if (ntups == 0)
- {
- PQclear(res);
- PQfinish(conn);
- return;
- }
-
/*
* Build qualified identifiers for each table, including the column list
* if given.
*/
- initPQExpBuffer(&buf);
- for (i = 0; i < ntups; i++)
+ if (*ntups > 0)
{
- appendPQExpBufferStr(&buf,
- fmtQualifiedId(PQgetvalue(res, i, 1),
- PQgetvalue(res, i, 0)));
+ initPQExpBuffer(&buf);
+ for (i = 0; i < *ntups; i++)
+ {
+ appendPQExpBufferStr(&buf,
+ fmtQualifiedId(PQgetvalue(res, i, 1),
+ PQgetvalue(res, i, 0)));
- if (objects_listed && !PQgetisnull(res, i, 2))
- appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
+ if (objects_listed && !PQgetisnull(res, i, 2))
+ appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
- simple_string_list_append(&dbtables, buf.data);
- resetPQExpBuffer(&buf);
+ simple_string_list_append(dbtables, buf.data);
+ resetPQExpBuffer(&buf);
+ }
+ termPQExpBuffer(&buf);
}
- termPQExpBuffer(&buf);
PQclear(res);
+ return dbtables;
+}
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ int ntups;
+ bool failed = false;
+ const char *initcmd;
+ SimpleStringList *dbtables;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+
+ dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
+
+ /*
+ * If no rows are returned, there are no matching tables, so we are done.
+ */
+ if (ntups == 0)
+ {
+ PQfinish(conn);
+ return;
+ }
+
+
/*
* Ensure concurrentCons is sane. If there are more connections than
* vacuumable relations, we don't need to use them all.
@@ -837,7 +859,7 @@ vacuum_one_database(ConnParams *cparams,
initPQExpBuffer(&sql);
- cell = dbtables.head;
+ cell = dbtables->head;
do
{
const char *tabname = cell->val;
--
2.47.1
[text/x-patch] v35-0012-Add-force-analyze-to-vacuumdb.patch (12.1K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/12-v35-0012-Add-force-analyze-to-vacuumdb.patch)
download | inline diff:
From 7626e26bb4a2fe71a34188e40167dcb5a38cea71 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Fri, 8 Nov 2024 12:27:50 -0500
Subject: [PATCH v35 12/12] Add --force-analyze to vacuumdb.
The vacuumdb options of --analyze-in-stages and --analyze-only are often
used after a restore from a dump or a pg_upgrade to quickly rebuild
stats on a databse.
However, now that stats are imported in most (but not all) cases,
running either of these commands will be at least partially redundant,
and will overwrite the stats that were just imported, which is a big
POLA violation.
We could add a new option such as --analyze-missing-in-stages, but that
wouldn't help the userbase that grown accustomed to running
--analyze-in-stages after an upgrade.
The least-bad option to handle the situation is to change the behavior
of --analyze-only and --analyze-in-stages to only analyze tables which
were missing stats before the vacuumdb started, but offer the
--force-analyze flag to restore the old behavior for those who truly
wanted it.
---
src/bin/scripts/t/100_vacuumdb.pl | 6 +-
src/bin/scripts/vacuumdb.c | 91 ++++++++++++++++++++++++-------
doc/src/sgml/ref/vacuumdb.sgml | 48 ++++++++++++++++
3 files changed, 125 insertions(+), 20 deletions(-)
diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl
index 1a2bcb4959..2d669391fe 100644
--- a/src/bin/scripts/t/100_vacuumdb.pl
+++ b/src/bin/scripts/t/100_vacuumdb.pl
@@ -127,9 +127,13 @@ $node->issues_sql_like(
qr/statement: VACUUM \(SKIP_DATABASE_STATS, ANALYZE\) public.vactable\(a, b\);/,
'vacuumdb --analyze with complete column list');
$node->issues_sql_like(
+ [ 'vacuumdb', '--analyze-only', '--force-analyze', '--table', 'vactable(b)', 'postgres' ],
+ qr/statement: ANALYZE public.vactable\(b\);/,
+ 'vacuumdb --analyze-only --force-analyze with partial column list');
+$node->issues_sql_unlike(
[ 'vacuumdb', '--analyze-only', '--table', 'vactable(b)', 'postgres' ],
qr/statement: ANALYZE public.vactable\(b\);/,
- 'vacuumdb --analyze-only with partial column list');
+ 'vacuumdb --analyze-only --force-analyze with partial column list skipping vacuumed tables');
$node->command_checks_all(
[ 'vacuumdb', '--analyze', '--table', 'vacview', 'postgres' ],
0,
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index b13f3c4224..1aa5c46af5 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -25,6 +25,7 @@
#include "fe_utils/query_utils.h"
#include "fe_utils/simple_list.h"
#include "fe_utils/string_utils.h"
+#include "pqexpbuffer.h"
/* vacuum options controlled by user flags */
@@ -47,6 +48,8 @@ typedef struct vacuumingOptions
bool process_main;
bool process_toast;
bool skip_database_stats;
+ bool analyze_in_stages;
+ bool force_analyze;
char *buffer_usage_limit;
} vacuumingOptions;
@@ -75,7 +78,6 @@ static void vacuum_one_database(ConnParams *cparams,
static void vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet);
@@ -140,6 +142,7 @@ main(int argc, char *argv[])
{"no-process-toast", no_argument, NULL, 11},
{"no-process-main", no_argument, NULL, 12},
{"buffer-usage-limit", required_argument, NULL, 13},
+ {"force-analyze", no_argument, NULL, 14},
{NULL, 0, NULL, 0}
};
@@ -156,7 +159,6 @@ main(int argc, char *argv[])
bool echo = false;
bool quiet = false;
vacuumingOptions vacopts;
- bool analyze_in_stages = false;
SimpleStringList objects = {NULL, NULL};
int concurrentCons = 1;
int tbl_count = 0;
@@ -170,6 +172,8 @@ main(int argc, char *argv[])
vacopts.do_truncate = true;
vacopts.process_main = true;
vacopts.process_toast = true;
+ vacopts.force_analyze = false;
+ vacopts.analyze_in_stages = false;
pg_logging_init(argv[0]);
progname = get_progname(argv[0]);
@@ -251,7 +255,7 @@ main(int argc, char *argv[])
maintenance_db = pg_strdup(optarg);
break;
case 3:
- analyze_in_stages = vacopts.analyze_only = true;
+ vacopts.analyze_in_stages = vacopts.analyze_only = true;
break;
case 4:
vacopts.disable_page_skipping = true;
@@ -287,6 +291,9 @@ main(int argc, char *argv[])
case 13:
vacopts.buffer_usage_limit = escape_quotes(optarg);
break;
+ case 14:
+ vacopts.force_analyze = true;
+ break;
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -372,6 +379,14 @@ main(int argc, char *argv[])
pg_fatal("cannot use the \"%s\" option with the \"%s\" option",
"buffer-usage-limit", "full");
+ /*
+ * --force-analyze is only valid when used with --analyze-only, -analyze,
+ * or --analyze-in-stages
+ */
+ if (vacopts.force_analyze && !vacopts.analyze_only && !vacopts.analyze_in_stages)
+ pg_fatal("can only use the \"%s\" option with \"%s\" or \"%s\"",
+ "--force-analyze", "-Z/--analyze-only", "--analyze-in-stages");
+
/* fill cparams except for dbname, which is set below */
cparams.pghost = host;
cparams.pgport = port;
@@ -390,7 +405,6 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
vacuum_all_databases(&cparams, &vacopts,
- analyze_in_stages,
&objects,
concurrentCons,
progname, echo, quiet);
@@ -413,20 +427,27 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts.analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectDatabase(&cparams, progname, echo, false, true);
check_conn_options(conn, &vacopts);
print_processing_notice(conn, stage, progname, quiet);
found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
+ vacuum_one_database(&cparams, &vacopts, stage,
+ &objects,
+ concurrentCons,
+ conn,
+ found_objects,
+ ntup,
+ progname, echo, quiet);
- if (analyze_in_stages)
+ if (stage != ANALYZE_NO_STAGE)
{
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
/* the last pass disconnected the conn */
- if (stage > 0)
- conn = connectDatabase(&cparams, progname, echo, false, true);
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(&cparams, &vacopts, stage,
&objects,
@@ -437,14 +458,6 @@ main(int argc, char *argv[])
progname, echo, quiet);
}
}
- else
- vacuum_one_database(&cparams, &vacopts, stage,
- &objects,
- concurrentCons,
- conn,
- found_objects,
- ntup,
- progname, echo, quiet);
}
exit(0);
@@ -763,6 +776,47 @@ generate_catalog_list(PGconn *conn,
vacopts->min_mxid_age);
}
+ /*
+ * If this query is for an analyze-only or analyze-in-stages, two
+ * upgrade-centric operations, and force-analyze is NOT set, then
+ * exclude any relations that already have their full compliment
+ * of attribute stats and extended stats.
+ *
+ *
+ */
+ if ((vacopts->analyze_only || vacopts->analyze_in_stages) &&
+ !vacopts->force_analyze)
+ {
+ /*
+ * The pg_class in question has no pg_statistic rows representing
+ * user-visible columns that lack a corresponding pg_statitic row.
+ * Currently no differentiation is made for whether the
+ * pg_statistic.stainherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_attribute AS a\n"
+ " WHERE a.attrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND a.attnum OPERATOR(pg_catalog.>) 0 AND NOT a.attisdropped\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic AS s\n"
+ " WHERE s.starelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND s.staattnum OPERATOR(pg_catalog.=) a.attnum))\n");
+
+ /*
+ * The pg_class entry has no pg_statistic_ext rows that lack a corresponding
+ * pg_statistic_ext_data row. Currently no differentiation is made for whether
+ * pg_statistic_exta_data.stxdinherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext AS e\n"
+ " WHERE e.stxrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext_data AS d\n"
+ " WHERE d.stxoid OPERATOR(pg_catalog.=) e.oid))\n");
+ }
+
/*
* Execute the catalog query. We use the default search_path for this
* query for consistency with table lookups done elsewhere by the user.
@@ -959,7 +1013,6 @@ finish:
static void
vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet)
@@ -972,7 +1025,7 @@ vacuum_all_databases(ConnParams *cparams,
SimpleStringList **found_objects;
int *num_tuples;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts->analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml
index 66fccb30a2..00ec927606 100644
--- a/doc/src/sgml/ref/vacuumdb.sgml
+++ b/doc/src/sgml/ref/vacuumdb.sgml
@@ -425,6 +425,12 @@ PostgreSQL documentation
<para>
Only calculate statistics for use by the optimizer (no vacuum).
</para>
+ <para>
+ By default, this operation excludes relations that already have
+ statistics generated. If the option <option>--force-analyze</option>
+ is also specified, then relations with existing stastistics are not
+ excluded.
+ </para>
</listitem>
</varlistentry>
@@ -439,6 +445,47 @@ PostgreSQL documentation
to produce usable statistics faster, and subsequent stages build the
full statistics.
</para>
+ <para>
+ This option is intended to be run after a <command>pg_upgrade</command>
+ to generate statistics for relations that have no stistatics or incomplete
+ statistics (such as those with extended statistics objects, which are not
+ imported on upgrade).
+ </para>
+ <para>
+ If the option <option>--force-analyze</option> is also specified, then
+ relations with existing stastistics are not excluded.
+ This option is only useful to analyze a database that currently has
+ no statistics or has wholly incorrect ones, such as if it is newly
+ populated from a restored dump.
+ Be aware that running with this option combinationin a database with
+ existing statistics may cause the query optimizer choices to become
+ transiently worse due to the low statistics targets of the early
+ stages.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--force-analyze</option></term>
+ <listitem>
+ <para>
+ This option can only be used if either <option>--analyze-only</option>
+ or <option>--analyze-in-stages</option> is specified. It modifies those
+ options to not filter out relations that already have statistics.
+ </para>
+ <para>
+
+ Only calculate statistics for use by the optimizer (no vacuum),
+ like <option>--analyze-only</option>. Run three
+ stages of analyze; the first stage uses the lowest possible statistics
+ target (see <xref linkend="guc-default-statistics-target"/>)
+ to produce usable statistics faster, and subsequent stages build the
+ full statistics.
+ </para>
+
+ <para>
+ This option was created
+ </para>
<para>
This option is only useful to analyze a database that currently has
@@ -452,6 +499,7 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+
<varlistentry>
<term><option>-?</option></term>
<term><option>--help</option></term>
--
2.47.1
[text/x-patch] v35-0011-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch (1.5K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/13-v35-0011-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch)
download | inline diff:
From 4fd29dd1ff736b6508f85e63273c0fb4fdde8625 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 7 Nov 2024 18:06:52 -0500
Subject: [PATCH v35 11/12] Add issues_sql_unlike, opposite of issues_sql_like
This is the same as issues_sql_like(), but the specified text is
prohibited from being in the output rather than required.
This became necessary to test that a command-line filter was in fact
filtering out certain output that a prior test required.
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 25 ++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 508e5e3917..1281071479 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2799,6 +2799,31 @@ sub issues_sql_like
}
=pod
+=item $node->issues_sql_unlike(cmd, prohibited_sql, test_name)
+
+Run a command on the node, then verify that $prohibited_sql does not appear
+in the server log file.
+
+=cut
+
+sub issues_sql_unlike
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+ my ($self, $cmd, $prohibited_sql, $test_name) = @_;
+
+ local %ENV = $self->_get_env();
+
+ my $log_location = -s $self->logfile;
+
+ my $result = PostgreSQL::Test::Utils::run_log($cmd);
+ ok($result, "@$cmd exit code 0");
+ my $log =
+ PostgreSQL::Test::Utils::slurp_file($self->logfile, $log_location);
+ unlike($log, $prohibited_sql, "$test_name: SQL found in server log");
+ return;
+}
+
=item $node->log_content()
--
2.47.1
[text/x-patch] v35-0010-preserve-catalog-lists-across-staged-runs.patch (6.7K, ../../CADkLM=eD=k5WFsWK4pCm5j3LjHN7KYj+0USz8L-dN9xO0J2UFQ@mail.gmail.com/14-v35-0010-preserve-catalog-lists-across-staged-runs.patch)
download | inline diff:
From c4b6a5d869f255ee9f5d853a25cdb2ee225c9193 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 21:30:49 -0500
Subject: [PATCH v35 10/12] preserve catalog lists across staged runs
---
src/bin/scripts/vacuumdb.c | 113 ++++++++++++++++++++++++++-----------
1 file changed, 80 insertions(+), 33 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 36f4796db0..b13f3c4224 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -68,6 +68,9 @@ static void vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet);
static void vacuum_all_databases(ConnParams *cparams,
@@ -83,6 +86,14 @@ static void prepare_vacuum_command(PQExpBuffer sql, int serverVersion,
static void run_vacuum_command(PGconn *conn, const char *sql, bool echo,
const char *table);
+static void check_conn_options(PGconn *conn, vacuumingOptions *vacopts);
+
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet);
+
+static SimpleStringList * generate_catalog_list(PGconn *conn, vacuumingOptions *vacopts,
+ SimpleStringList *objects, bool echo, int *ntups);
+
static void help(const char *progname);
void check_objfilter(void);
@@ -386,6 +397,11 @@ main(int argc, char *argv[])
}
else
{
+ PGconn *conn;
+ int ntup;
+ SimpleStringList *found_objects;
+ int stage;
+
if (dbname == NULL)
{
if (getenv("PGDATABASE"))
@@ -397,25 +413,37 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ check_conn_options(conn, &vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
if (analyze_in_stages)
{
- int stage;
-
for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
{
- vacuum_one_database(&cparams, &vacopts,
- stage,
+ /* the last pass disconnected the conn */
+ if (stage > 0)
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
}
else
- vacuum_one_database(&cparams, &vacopts,
- ANALYZE_NO_STAGE,
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
@@ -791,16 +819,17 @@ vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet)
{
PQExpBufferData sql;
- PGconn *conn;
SimpleStringListCell *cell;
ParallelSlotArray *sa;
- int ntups;
bool failed = false;
const char *initcmd;
- SimpleStringList *dbtables;
+
const char *stage_commands[] = {
"SET default_statistics_target=1; SET vacuum_cost_delay=0;",
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
@@ -810,13 +839,6 @@ vacuum_one_database(ConnParams *cparams,
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
-
/*
* If no rows are returned, there are no matching tables, so we are done.
*/
@@ -928,7 +950,7 @@ finish:
}
/*
- * Vacuum/analyze all connectable databases.
+ * Vacuum/analyze all ccparams->override_dbname = PQgetvalue(result, i, 0);onnectable databases.
*
* In analyze-in-stages mode, we process all databases in one stage before
* moving on to the next stage. That ensure minimal stats are available
@@ -944,8 +966,13 @@ vacuum_all_databases(ConnParams *cparams,
{
PGconn *conn;
PGresult *result;
- int stage;
int i;
+ int stage;
+
+ SimpleStringList **found_objects;
+ int *num_tuples;
+
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
@@ -953,7 +980,33 @@ vacuum_all_databases(ConnParams *cparams,
echo);
PQfinish(conn);
- if (analyze_in_stages)
+ /*
+ * connect to each database, check validity of options,
+ * build the list of found objects per database,
+ * and run the first/only vacuum stage
+ */
+ found_objects = palloc(PQntuples(result) * sizeof(SimpleStringList *));
+ num_tuples = palloc(PQntuples(result) * sizeof (int));
+
+ for (i = 0; i < PQntuples(result); i++)
+ {
+ cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects[i] = generate_catalog_list(conn, vacopts, objects, echo, &num_tuples[i]);
+
+ vacuum_one_database(cparams, vacopts,
+ stage,
+ objects,
+ concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
+ progname, echo, quiet);
+ }
+
+ if (stage != ANALYZE_NO_STAGE)
{
/*
* When analyzing all databases in stages, we analyze them all in the
@@ -963,35 +1016,29 @@ vacuum_all_databases(ConnParams *cparams,
* This means we establish several times as many connections, but
* that's a secondary consideration.
*/
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
for (i = 0; i < PQntuples(result); i++)
{
cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(cparams, vacopts,
stage,
objects,
concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
progname, echo, quiet);
}
}
}
- else
- {
- for (i = 0; i < PQntuples(result); i++)
- {
- cparams->override_dbname = PQgetvalue(result, i, 0);
-
- vacuum_one_database(cparams, vacopts,
- ANALYZE_NO_STAGE,
- objects,
- concurrentCons,
- progname, echo, quiet);
- }
- }
PQclear(result);
+ pfree(found_objects);
+ pfree(num_tuples);
}
/*
--
2.47.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-13 05:22 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2024-12-20 05:23 ` Jeff Davis <[email protected]>
2024-12-20 06:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-21 00:16 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-26 18:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
0 siblings, 3 replies; 81+ messages in thread
From: Jeff Davis @ 2024-12-20 05:23 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Fri, 2024-12-13 at 00:22 -0500, Corey Huinker wrote:
> Per offline conversation with Jeff, adding a --no-schema to pg_dump
> option both for completeness (we already have --no-data and --no-
> statistics), but users who previously got the effect of --no-schema
> did so by specifying --data-only, which suppresses statistics as
> well.
>
> 0001-0005 - changes to pg_dump/pg_upgrade
Attached is a version 36j where I consolidated these patches and
cleaned up the documentation. It doesn't make a lot of sense to commit
them separately, because as soon as the pg_dump changes are there, the
pg_upgrade test starts showing a difference until it starts using the -
-no-data option.
The biggest functional change is the way dependencies are handled for
matview stats. Materialized views ordinarily end up in
SECITON_PRE_DATA, but in some cases they can be postponed to
SECTION_POST_DATA. You solved that by always putting the matview stats
in SECTION_POST_DATA.
I took a different approach here and, when the matview is postponed,
also postpone the matview stats. It's slightly more code, but it felt
closer to the rest of the structure, where postponing is a special case
(that we might be able to remove in the future).
Regards,
Jeff Davis
Attachments:
[text/x-patch] v36j-0001-Dump-table-index-stats-in-pg_dump-and-transfer-.patch (49.7K, ../../[email protected]/2-v36j-0001-Dump-table-index-stats-in-pg_dump-and-transfer-.patch)
download | inline diff:
From 136ed6586bed7b9cd4415fc882bb30a19fa2063e Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v36j] Dump table/index stats in pg_dump and transfer in
pg_upgrade.
For each table/matview/index dumped, pg_dump will generate a statement
that calls pg_set_relation_stats(), and a series of statements that
call pg_set_attribute_stats(), one per attribute.
During restore, these statements will recreate the statistics of the
source system in the destination system.
For pg_dump, adds the command-line options --statistics-only (-X),
--no-schema, --no-statistics, and --no-data to enable the various
combinations of schema, statistics, and data.
Table statistics are dumped in the data section. Index and
Materialized View statistics are dumped in the post-data section.
Add options --with-statistics/--no-statistics to pg_upgrade to
enable/disable transferring of statistics to the upgraded cluster. The
default is --with-statistics.
Author: Corey Huinker
Discussion: https://postgr.es/m/CADkLM=fyJ-Y-DNk1aW09btZYdXDXS79xT8oFPTQ6sspWHaqdog@mail.gmail.com
---
doc/src/sgml/ref/pg_dump.sgml | 77 ++++--
doc/src/sgml/ref/pg_dumpall.sgml | 38 +++
doc/src/sgml/ref/pg_restore.sgml | 47 +++-
doc/src/sgml/ref/pgupgrade.sgml | 17 ++
src/bin/pg_dump/pg_backup.h | 10 +-
src/bin/pg_dump/pg_backup_archiver.c | 8 +
src/bin/pg_dump/pg_dump.c | 382 ++++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 9 +
src/bin/pg_dump/pg_dump_sort.c | 33 ++-
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 27 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
src/bin/pg_upgrade/dump.c | 6 +-
src/bin/pg_upgrade/option.c | 12 +
src/bin/pg_upgrade/pg_upgrade.h | 1 +
src/tools/pgindent/typedefs.list | 1 +
16 files changed, 655 insertions(+), 36 deletions(-)
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d66e901f51b..9340aa70a77 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,14 +123,9 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
-
- <para>
- This option is similar to, but for historical reasons not identical
- to, specifying <option>--section=data</option>.
- </para>
</listitem>
</varlistentry>
@@ -141,13 +136,11 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
- switch is therefore only useful to add large objects to dumps
- where a specific schema or table has been requested. Note that
- large objects are considered data and therefore will be included when
- <option>--data-only</option> is used, but not
- when <option>--schema-only</option> is.
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, <option>--statistics-only</option>, or
+ <option>--no-data</option> is specified. The <option>-b</option>
+ switch is therefore only useful to add large objects to dumps where a
+ specific schema or table has been requested.
</para>
</listitem>
</varlistentry>
@@ -516,10 +509,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +646,17 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -824,16 +829,17 @@ PostgreSQL documentation
<term><option>--exclude-table-data=<replaceable class="parameter">pattern</replaceable></option></term>
<listitem>
<para>
- Do not dump data for any tables matching <replaceable
- class="parameter">pattern</replaceable>. The pattern is
- interpreted according to the same rules as for <option>-t</option>.
+ Do not dump data or statistics for any tables matching <replaceable
+ class="parameter">pattern</replaceable>. The pattern is interpreted
+ according to the same rules as for <option>-t</option>.
<option>--exclude-table-data</option> can be given more than once to
- exclude tables matching any of several patterns. This option is
- useful when you need the definition of a particular table even
- though you do not need the data in it.
+ exclude tables matching any of several patterns. This option is useful
+ when you need the definition of a particular table even though you do
+ not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1080,6 +1086,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -1089,6 +1104,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -1098,6 +1122,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index 014f2792589..d423153a93a 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -265,6 +265,17 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--binary-upgrade</option></term>
<listitem>
@@ -422,6 +433,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -447,6 +467,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -456,6 +485,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719e..3c381db1aa7 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema (data definitions) or data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -681,6 +692,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore data, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-data-for-failed-tables</option></term>
<listitem>
@@ -713,6 +734,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema (data definitions), even if
+ the archive contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -723,6 +754,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml
index 4777381dac2..0a36db40e97 100644
--- a/doc/src/sgml/ref/pgupgrade.sgml
+++ b/doc/src/sgml/ref/pgupgrade.sgml
@@ -145,6 +145,23 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-statistics</option></term>
+ <listitem>
+ <para>
+ Restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
<varlistentry>
<term><option>-o</option> <replaceable class="parameter">options</replaceable></term>
<term><option>--old-options</option> <replaceable class="parameter">options</replaceable></term>
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b29..6e17bd28934 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,8 +110,11 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
+ int no_statistics; /* Skip statistics import */
int no_subscriptions; /* Skip subscription entries */
int strict_names;
@@ -160,6 +163,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -179,8 +183,11 @@ typedef struct _dumpOptions
int column_inserts;
int if_exists;
int no_comments;
- int no_security_labels;
+ int no_data;
int no_publications;
+ int no_schema;
+ int no_security_labels;
+ int no_statistics;
int no_subscriptions;
int no_toast_compression;
int no_unlogged_table_data;
@@ -208,6 +215,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844c..d62f419560d 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -186,6 +186,9 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
@@ -2962,6 +2965,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's a stats dump, maybe ignore it */
+ if (ropt->no_statistics && strcmp(te->desc, "STATISTICS") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +2998,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 19969e400fc..e6eff0c3297 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -430,6 +430,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -466,6 +467,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -492,8 +494,11 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -539,7 +544,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:PRsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -613,6 +618,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -784,6 +793,17 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -798,8 +818,9 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
@@ -1099,6 +1120,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1177,7 +1199,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1190,11 +1212,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1219,8 +1242,11 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
+ printf(_(" --no-schema do not dump schema\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6777,6 +6803,43 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if (relkind == RELKIND_RELATION ||
+ relkind == RELKIND_PARTITIONED_TABLE ||
+ relkind == RELKIND_MATVIEW ||
+ relkind == RELKIND_INDEX ||
+ relkind == RELKIND_PARTITIONED_INDEX)
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+ info->postponed_def = false;
+
+ return info;
+ }
+
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7154,6 +7217,7 @@ getTables(Archive *fout, int *numTables)
/* Tables have data */
tblinfo[i].dobj.components |= DUMP_COMPONENT_DATA;
+ tblinfo[i].dobj.components |= DUMP_COMPONENT_STATISTICS;
/* Mark whether table has an ACL */
if (!PQgetisnull(res, i, i_relacl))
@@ -7202,6 +7266,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7648,11 +7714,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7675,7 +7744,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7709,6 +7785,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10296,6 +10374,287 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * Decide which section to use based on the relkind of the parent object.
+ *
+ * NB: materialized views may be postponed from SECTION_PRE_DATA to
+ * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the
+ * matview stats will also be postponed to SECTION_POST_DATA. See
+ * repairMatViewBoundaryMultiLoop().
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+ switch (rsinfo->relkind)
+ {
+ case RELKIND_RELATION:
+ case RELKIND_PARTITIONED_TABLE:
+ case RELKIND_MATVIEW:
+ return SECTION_DATA;
+ case RELKIND_INDEX:
+ case RELKIND_PARTITIONED_INDEX:
+ return SECTION_POST_DATA;
+ default:
+ pg_fatal("cannot dump statistics for relation kind '%c'",
+ rsinfo->relkind);
+ }
+
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = rsinfo->postponed_def ?
+ SECTION_POST_DATA : statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = dobj->dependencies,
+ .nDeps = dobj->nDeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10744,6 +11103,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -18970,6 +19332,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 9c5ddd20cf7..b107a163d28 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -84,6 +84,7 @@ typedef enum
DO_PUBLICATION_TABLE_IN_SCHEMA,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
+ DO_REL_STATS,
} DumpableObjectType;
/*
@@ -109,6 +110,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -429,6 +431,13 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'v', 'c', etc */
+ bool postponed_def;
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index 3c8f2eb808d..400236ca95e 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -801,11 +801,22 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj,
{
/* remove boundary's dependency on object after it in loop */
removeObjectDependency(boundaryobj, nextobj->dumpId);
- /* if that object is a matview, mark it as postponed into post-data */
+
+ /*
+ * If that object is a matview or matview stats, mark it as postponed into
+ * post-data.
+ */
if (nextobj->objType == DO_TABLE)
{
TableInfo *nextinfo = (TableInfo *) nextobj;
+ if (nextinfo->relkind == RELKIND_MATVIEW)
+ nextinfo->postponed_def = true;
+ }
+ else if (nextobj->objType == DO_REL_STATS)
+ {
+ RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj;
+
if (nextinfo->relkind == RELKIND_MATVIEW)
nextinfo->postponed_def = true;
}
@@ -1018,6 +1029,21 @@ repairDependencyLoop(DumpableObject **loop,
{
DumpableObject *nextobj;
+ nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
+ repairMatViewBoundaryMultiLoop(loop[j], nextobj);
+ return;
+ }
+ }
+ }
+ else if (loop[i]->objType == DO_REL_STATS &&
+ ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW)
+ {
+ for (j = 0; j < nLoop; j++)
+ {
+ if (loop[j]->objType == DO_POST_DATA_BOUNDARY)
+ {
+ DumpableObject *nextobj;
+
nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
repairMatViewBoundaryMultiLoop(loop[j], nextobj);
return;
@@ -1500,6 +1526,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 9a04e51c81a..62e2766c094 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 88ae39d938a..c98a8d08dd4 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,7 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -71,8 +72,11 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_comments = 0;
+ static int no_data = 0;
static int no_publications = 0;
+ static int no_schema = 0;
static int no_security_labels = 0;
+ static int no_statistics = 0;
static int no_subscriptions = 0;
static int strict_names = 0;
bool data_only = false;
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'P'},
/*
* the following options don't have an equivalent short option letter
@@ -124,8 +129,11 @@ main(int argc, char **argv)
{"transaction-size", required_argument, NULL, 5},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
{"filter", required_argument, NULL, 4},
@@ -271,6 +279,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +355,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +378,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpSchema = (!no_schema && !data_only && !statistics_only);
+ opts->dumpData = (!no_data && !schema_only && !statistics_only);
+ opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +392,8 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
@@ -484,6 +503,7 @@ usage(const char *progname)
printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n"));
printf(_(" -T, --trigger=NAME restore named trigger\n"));
printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only restore only the statistics, not schema or data\n"));
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
@@ -491,10 +511,13 @@ usage(const char *progname)
" in FILENAME\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comment commands\n"));
+ printf(_(" --no-data do not restore data\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index b9d13a0e1de..76ebf15f55b 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8345f55be8a..954e7bacf4f 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 6f41d63eed4..e2f564f5d58 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -306,7 +316,9 @@ usage(void)
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
+ printf(_(" --no-statisttics do not import statistics from old cluster\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statisttics import statistics from old cluster (default)\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 53f693c2d4b..371f9a2d063 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index fbdb932e6b6..d5aa045db4f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2393,6 +2393,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelStatsInfo
RelToCheck
RelToCluster
RelabelType
--
2.34.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-13 05:22 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-20 05:23 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2024-12-20 06:00 ` Corey Huinker <[email protected]>
2 siblings, 0 replies; 81+ messages in thread
From: Corey Huinker @ 2024-12-20 06:00 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
> The biggest functional change is the way dependencies are handled for
> matview stats. Materialized views ordinarily end up in
> SECITON_PRE_DATA, but in some cases they can be postponed to
> SECTION_POST_DATA. You solved that by always putting the matview stats
> in SECTION_POST_DATA.
>
Accurate.
> I took a different approach here and, when the matview is postponed,
> also postpone the matview stats. It's slightly more code, but it felt
> closer to the rest of the structure, where postponing is a special case
> (that we might be able to remove in the future).
+1. The fact that this quirk was a knock-on effect of the postponing-quirk,
which could go away, makes this change compelling.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-13 05:22 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-20 05:23 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2024-12-21 00:16 ` Jeff Davis <[email protected]>
2024-12-21 10:11 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2 siblings, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2024-12-21 00:16 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Thu, 2024-12-19 at 21:23 -0800, Jeff Davis wrote:
> > 0001-0005 - changes to pg_dump/pg_upgrade
>
> Attached is a version 36j...
The testing can use some work here. I noticed that if I take out the
stats entirely, the tests still pass, because pg_upgrade still gets the
same before/after result.
Also, we need some testing of the output and ordering of pg_dump.
Granted, in most cases problems would result in errors during the
reload. But we have those tests for other kinds of objects, so we
should have the tests for stats, too.
I like the description "STATISTICS DATA" because it differentiates from
the extended stats definitions. It might be worth differentiating
between "RELATION STATISTICS DATA" and "ATTRIBUTE STATISTICS DATA" but
I'm not sure if there's value in that.
But how did you determine what to use for the .tag and prefix? In the
output, it uses the form:
Name: STATISTICS DATA <name>; Type: STATISTICS DATA; ...
Should that be:
Name: <name>; Type: STATISTICS DATA; ...
Or:
Data for Name: ...; Name: ...; Type: STATISTICS DATA; ...
Or:
Statistics for Name: ...; Name: ...; Type: STATISTICS DATA; ...
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-13 05:22 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-20 05:23 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-21 00:16 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2024-12-21 10:11 ` Corey Huinker <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Corey Huinker @ 2024-12-21 10:11 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
> I like the description "STATISTICS DATA" because it differentiates from
>
You have Tomas to thank for that:
https://www.postgresql.org/message-id/bf724b21-914a-4497-84e3-49944f9776f6%40enterprisedb.com
the extended stats definitions. It might be worth differentiating
> between "RELATION STATISTICS DATA" and "ATTRIBUTE STATISTICS DATA" but
> I'm not sure if there's value in that.
>
I have no objection to such a change, even if we don't currently have a use
for the differentiation, someone in the future might.
>
> But how did you determine what to use for the .tag and prefix? In the
> output, it uses the form:
>
It was the minimal change needed to meet Tomas's suggestion.
> Statistics for Name: ...; Name: ...; Type: STATISTICS DATA; ...
>
I like this one best, because it clarifies the meaning of STATISTICS DATA.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-13 05:22 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-20 05:23 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2024-12-26 18:45 ` Bruce Momjian <[email protected]>
2 siblings, 0 replies; 81+ messages in thread
From: Bruce Momjian @ 2024-12-26 18:45 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Corey Huinker <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Thu, Dec 19, 2024 at 09:23:20PM -0800, Jeff Davis wrote:
> On Fri, 2024-12-13 at 00:22 -0500, Corey Huinker wrote:
> > Per offline conversation with Jeff, adding a --no-schema to pg_dump
> > option both for completeness (we already have --no-data and --no-
> > statistics), but users who previously got the effect of --no-schema
> > did so by specifying --data-only, which suppresses statistics as
> > well.
> >
> > 0001-0005 - changes to pg_dump/pg_upgrade
>
> Attached is a version 36j where I consolidated these patches and
> cleaned up the documentation. It doesn't make a lot of sense to commit
> them separately, because as soon as the pg_dump changes are there, the
> pg_upgrade test starts showing a difference until it starts using the -
> -no-data option.
>
> The biggest functional change is the way dependencies are handled for
> matview stats. Materialized views ordinarily end up in
> SECITON_PRE_DATA, but in some cases they can be postponed to
> SECTION_POST_DATA. You solved that by always putting the matview stats
> in SECTION_POST_DATA.
>
> I took a different approach here and, when the matview is postponed,
> also postpone the matview stats. It's slightly more code, but it felt
> closer to the rest of the structure, where postponing is a special case
> (that we might be able to remove in the future).
I am confused by this:
Add options --with-statistics/--no-statistics to pg_upgrade
to enable/disable transferring of statistics to the upgraded
cluster. The default is --with-statistics.
If statistics is the default for pg_upgrade, why would we need a
--with-statistics option?
Also, I see a misspelling:
+ printf(_(" --no-statisttics do not import statistics from old cluster\n"));
--
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Do not let urgent matters crowd out time for investment in the future.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2024-12-26 18:54 ` Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Bruce Momjian @ 2024-12-26 18:54 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Jeff Davis <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Wed, Dec 11, 2024 at 10:49:53PM -0500, Corey Huinker wrote:
> From cf4e731db9ffaa4e89d7c5d14b32668529c8c89a Mon Sep 17 00:00:00 2001
> From: Corey Huinker <[email protected]>
> Date: Fri, 8 Nov 2024 12:27:50 -0500
> Subject: [PATCH v34 11/11] Add --force-analyze to vacuumdb.
>
> The vacuumdb options of --analyze-in-stages and --analyze-only are often
> used after a restore from a dump or a pg_upgrade to quickly rebuild
> stats on a databse.
>
> However, now that stats are imported in most (but not all) cases,
> running either of these commands will be at least partially redundant,
> and will overwrite the stats that were just imported, which is a big
> POLA violation.
>
> We could add a new option such as --analyze-missing-in-stages, but that
> wouldn't help the userbase that grown accustomed to running
> --analyze-in-stages after an upgrade.
>
> The least-bad option to handle the situation is to change the behavior
> of --analyze-only and --analyze-in-stages to only analyze tables which
> were missing stats before the vacuumdb started, but offer the
> --force-analyze flag to restore the old behavior for those who truly
> wanted it.
I am _again_ not happy with this part of the patch. Please reply to the
criticism in my November 19th email:
https://www.postgresql.org/message-id/[email protected]
rather than ignoring it and posting the same version of the patch.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Do not let urgent matters crowd out time for investment in the future.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
@ 2024-12-30 20:02 ` Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2024-12-30 20:02 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; Corey Huinker <[email protected]>; +Cc: jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Thu, 2024-12-26 at 13:54 -0500, Bruce Momjian wrote:
> I am _again_ not happy with this part of the patch. Please reply to
> the
> criticism in my November 19th email:
>
>
> https://www.postgresql.org/message-id/[email protected]
>
> rather than ignoring it and posting the same version of the patch.
I suggest that we make a new thread about the vacuumdb changes and
focus this thread and patch series on the pg_dump changes (and minor
flag adjustments to pg_upgrade).
Unless you think that the pg_dump changes should block on the vacuumdb
changes? In which case please let me know because the pg_dump changes
are otherwise close to commit.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2024-12-30 20:45 ` Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Bruce Momjian @ 2024-12-30 20:45 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Corey Huinker <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Nathan Bossart <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Mon, Dec 30, 2024 at 12:02:47PM -0800, Jeff Davis wrote:
> On Thu, 2024-12-26 at 13:54 -0500, Bruce Momjian wrote:
> > I am _again_ not happy with this part of the patch. Please reply to
> > the
> > criticism in my November 19th email:
> >
> >
> > https://www.postgresql.org/message-id/[email protected]
> >
> > rather than ignoring it and posting the same version of the patch.
>
> I suggest that we make a new thread about the vacuumdb changes and
> focus this thread and patch series on the pg_dump changes (and minor
> flag adjustments to pg_upgrade).
>
> Unless you think that the pg_dump changes should block on the vacuumdb
> changes? In which case please let me know because the pg_dump changes
> are otherwise close to commit.
I think that is a good idea. I don't see vacuumdb blocking this.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Do not let urgent matters crowd out time for investment in the future.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
@ 2025-01-06 21:27 ` Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Nathan Bossart @ 2025-01-06 21:27 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Mon, Dec 30, 2024 at 03:45:03PM -0500, Bruce Momjian wrote:
> On Mon, Dec 30, 2024 at 12:02:47PM -0800, Jeff Davis wrote:
>> I suggest that we make a new thread about the vacuumdb changes and
>> focus this thread and patch series on the pg_dump changes (and minor
>> flag adjustments to pg_upgrade).
>>
>> Unless you think that the pg_dump changes should block on the vacuumdb
>> changes? In which case please let me know because the pg_dump changes
>> are otherwise close to commit.
>
> I think that is a good idea. I don't see vacuumdb blocking this.
+1, I've been reviewing the vacuumdb portion and am planning to start a new
thread in the near future. IIUC the bulk of the vacuumdb changes are
relatively noncontroversial, we just haven't reached consensus on the user
interface.
--
nathan
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
@ 2025-01-07 06:18 ` Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-01-07 06:18 UTC (permalink / raw)
To: Nathan Bossart <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Jeff Davis <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
> +1, I've been reviewing the vacuumdb portion and am planning to start a new
> thread in the near future. IIUC the bulk of the vacuumdb changes are
> relatively noncontroversial, we just haven't reached consensus on the user
> interface.
>
> --
> nathan
>
Attached is the latest (and probably last) unified patchset before parts
get spun off into their own threads.
0001 - This is the unified changes to pg_dump, pg_restore, pg_dumpall, and
pg_upgrade.
It incorporates most of what Jeff changed when he unified v36j, with typo
fixes spotted by Bruce. There was interest in splitting STATISTICS DATA
into RELATION STATISTICS DATA and ATTRIBUTE STATISTICS DATA. To do that,
we'd need to create separate TOC entries, and that doesn't seem worth it to
me.
There was also interest in changing the prefix for STATISTICS DATA.
However, the only special case for prefixes currently relies on an isData
flag. Since there is no isStatistics flag, we would either have to create
one, or do strcmps on te->description looking for "STATISTICS DATA". It's
do-able, but I'm not sure it's worth it.
0002-0005 are for extended stats.
0002 - This makes the input function for pg_ndistinct functional.
0003 - This makes the input function for pg_dependencies functional.
0004 - Makes several static functions in attribute_stats.c public for use
by extended stats. One of those is get_stat_attr_type(), which in the last
patchset was modified to take an attribute name rather than attnum, thus
saving a syscache lookup. However, extended stats identifies attributes by
attnum not name, so that optimization had to be set aside, at least
temporarily.
0005 - These implement the functions pg_set_extended_stats(),
pg_clear_extended_stats(), and pg_restore_extended_stats() and behave like
their relation/attribute equivalents. If we can get these committed and
used by pg_dump, then we don't have to debate how to handle post-upgrade
steps for users who happen to have extended stats vs the approximately
99.75% of users who do not have extended stats.
0006-0011 - These are the currently on-ice vacuumdb changes. 0006-0009 are
"do no harm" reorganizations which add no functionality but make 0011
clearer. 0010 introduces a new test check to TAP. This group will be spun
out into their own thread where we can focus on what if any changes happen
to vacuumdb. Any changes to vacuumdb are dependent on 0001 being committed,
and any changes to vacuumdb would likewise be informed by the commit status
of 0002-0005.
Work still to be done, in the near and less-near term:
* Adding statistic import for extended statistics objects (i.e. CREATE
STATISTICS) to pg_dump. Blocked by 0002-0005.
* Implementing optional statistics import in postgres_fdw. Blocked by 0001.
Attachments:
[text/x-patch] v37-0004-Expose-attribute-statistics-functions-for-use-in.patch (4.9K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/3-v37-0004-Expose-attribute-statistics-functions-for-use-in.patch)
download | inline diff:
From 43a4e7891f939b68d7f5e8091e7df3af766fdd8c Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 26 Dec 2024 05:02:06 -0500
Subject: [PATCH v37 04/11] Expose attribute statistics functions for use in
extended_stats.
Many of the operations of attribute stats have analogous operations in
extended stats.
* get_attr_stat_type()
* init_empty_stats_tuple()
* text_to_stavalues()
* get_elem_stat_type()
---
src/include/statistics/statistics.h | 17 +++++++++++++++++
src/backend/statistics/attribute_stats.c | 24 +++++-------------------
2 files changed, 22 insertions(+), 19 deletions(-)
diff --git a/src/include/statistics/statistics.h b/src/include/statistics/statistics.h
index 7dd0f97554..f47f192dff 100644
--- a/src/include/statistics/statistics.h
+++ b/src/include/statistics/statistics.h
@@ -127,4 +127,21 @@ extern StatisticExtInfo *choose_best_statistics(List *stats, char requiredkind,
int nclauses);
extern HeapTuple statext_expressions_load(Oid stxoid, bool inh, int idx);
+extern void get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
+ Oid *atttypid, int32 *atttypmod,
+ char *atttyptype, Oid *atttypcoll,
+ Oid *eq_opr, Oid *lt_opr);
+extern void init_empty_stats_tuple(Oid reloid, int16 attnum, bool inherited,
+ Datum *values, bool *nulls, bool *replaces);
+
+extern void set_stats_slot(Datum *values, bool *nulls, bool *replaces,
+ int16 stakind, Oid staop, Oid stacoll,
+ Datum stanumbers, bool stanumbers_isnull,
+ Datum stavalues, bool stavalues_isnull);
+
+extern Datum text_to_stavalues(const char *staname, FmgrInfo *array_in, Datum d,
+ Oid typid, int32 typmod, int elevel, bool *ok);
+extern bool get_elem_stat_type(Oid atttypid, char atttyptype, int elevel,
+ Oid *elemtypid, Oid *elem_eq_opr);
+
#endif /* STATISTICS_H */
diff --git a/src/backend/statistics/attribute_stats.c b/src/backend/statistics/attribute_stats.c
index 94f7dd63a0..f617165386 100644
--- a/src/backend/statistics/attribute_stats.c
+++ b/src/backend/statistics/attribute_stats.c
@@ -78,23 +78,9 @@ static struct StatsArgInfo attarginfo[] =
static bool attribute_statistics_update(FunctionCallInfo fcinfo, int elevel);
static Node *get_attr_expr(Relation rel, int attnum);
-static void get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
- Oid *atttypid, int32 *atttypmod,
- char *atttyptype, Oid *atttypcoll,
- Oid *eq_opr, Oid *lt_opr);
-static bool get_elem_stat_type(Oid atttypid, char atttyptype, int elevel,
- Oid *elemtypid, Oid *elem_eq_opr);
-static Datum text_to_stavalues(const char *staname, FmgrInfo *array_in, Datum d,
- Oid typid, int32 typmod, int elevel, bool *ok);
-static void set_stats_slot(Datum *values, bool *nulls, bool *replaces,
- int16 stakind, Oid staop, Oid stacoll,
- Datum stanumbers, bool stanumbers_isnull,
- Datum stavalues, bool stavalues_isnull);
static void upsert_pg_statistic(Relation starel, HeapTuple oldtup,
Datum *values, bool *nulls, bool *replaces);
static bool delete_pg_statistic(Oid reloid, AttrNumber attnum, bool stainherit);
-static void init_empty_stats_tuple(Oid reloid, int16 attnum, bool inherited,
- Datum *values, bool *nulls, bool *replaces);
/*
* Insert or Update Attribute Statistics
@@ -502,7 +488,7 @@ get_attr_expr(Relation rel, int attnum)
/*
* Derive type information from the attribute.
*/
-static void
+void
get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
Oid *atttypid, int32 *atttypmod,
char *atttyptype, Oid *atttypcoll,
@@ -584,7 +570,7 @@ get_attr_stat_type(Oid reloid, AttrNumber attnum, int elevel,
/*
* Derive element type information from the attribute type.
*/
-static bool
+bool
get_elem_stat_type(Oid atttypid, char atttyptype, int elevel,
Oid *elemtypid, Oid *elem_eq_opr)
{
@@ -624,7 +610,7 @@ get_elem_stat_type(Oid atttypid, char atttyptype, int elevel,
* to false. If the resulting array contains NULLs, raise an error at elevel
* and set ok to false. Otherwise, set ok to true.
*/
-static Datum
+Datum
text_to_stavalues(const char *staname, FmgrInfo *array_in, Datum d, Oid typid,
int32 typmod, int elevel, bool *ok)
{
@@ -678,7 +664,7 @@ text_to_stavalues(const char *staname, FmgrInfo *array_in, Datum d, Oid typid,
* Find and update the slot with the given stakind, or use the first empty
* slot.
*/
-static void
+void
set_stats_slot(Datum *values, bool *nulls, bool *replaces,
int16 stakind, Oid staop, Oid stacoll,
Datum stanumbers, bool stanumbers_isnull,
@@ -802,7 +788,7 @@ delete_pg_statistic(Oid reloid, AttrNumber attnum, bool stainherit)
/*
* Initialize values and nulls for a new stats tuple.
*/
-static void
+void
init_empty_stats_tuple(Oid reloid, int16 attnum, bool inherited,
Datum *values, bool *nulls, bool *replaces)
{
--
2.47.1
[text/x-patch] v37-0002-Add-working-input-function-for-pg_ndistinct.patch (10.4K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/4-v37-0002-Add-working-input-function-for-pg_ndistinct.patch)
download | inline diff:
From 658234311aaba0063fabc3d6bfb76f3f5f28012b Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Tue, 17 Dec 2024 03:30:55 -0500
Subject: [PATCH v37 02/11] Add working input function for pg_ndistinct.
This is needed to import extended statistics.
---
src/backend/statistics/mvdistinct.c | 270 +++++++++++++++++++++++-
src/test/regress/expected/stats_ext.out | 7 +
src/test/regress/sql/stats_ext.sql | 3 +
3 files changed, 274 insertions(+), 6 deletions(-)
diff --git a/src/backend/statistics/mvdistinct.c b/src/backend/statistics/mvdistinct.c
index 7e7a63405c..13ca2a9fd1 100644
--- a/src/backend/statistics/mvdistinct.c
+++ b/src/backend/statistics/mvdistinct.c
@@ -27,10 +27,18 @@
#include "catalog/pg_statistic_ext.h"
#include "catalog/pg_statistic_ext_data.h"
+#include "common/jsonapi.h"
+#include "fmgr.h"
#include "lib/stringinfo.h"
+#include "mb/pg_wchar.h"
+#include "nodes/miscnodes.h"
+#include "nodes/pg_list.h"
#include "statistics/extended_stats_internal.h"
#include "statistics/statistics.h"
+#include "utils/builtins.h"
+#include "utils/float.h"
#include "utils/fmgrprotos.h"
+#include "utils/palloc.h"
#include "utils/syscache.h"
#include "utils/typcache.h"
#include "varatt.h"
@@ -328,21 +336,271 @@ statext_ndistinct_deserialize(bytea *data)
return ndistinct;
}
+typedef struct
+{
+ const char *str;
+ bool found_only_object;
+ List *distinct_items;
+ Node *escontext;
+
+ MVNDistinctItem *current_item;
+} ndistinctParseState;
+
+/*
+ * Invoked at the start of each object in the JSON document.
+ * The entire JSON document should be one object with no sub-objects.
+ *
+ * If we're anywhere else in the document, it's an error.
+ */
+static JsonParseErrorType
+ndistinct_object_start(void *state)
+{
+ ndistinctParseState *parse = state;
+
+ if (parse->found_only_object == true)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_ndistinct: \"%s\"", parse->str),
+ errdetail("Must begin with \"{\"")));
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ parse->found_only_object = true;
+ return JSON_SUCCESS;
+}
+
+/*
+ * ndsitinct input format does not have arrays, so any array elements encountered
+ * are an error.
+ */
+static JsonParseErrorType
+ndistinct_array_start(void *state)
+{
+ ndistinctParseState *parse = state;
+
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_ndistinct: \"%s\"", parse->str),
+ errdetail("All ndistinct count values are scalar doubles.")));
+ return JSON_SEM_ACTION_FAILED;
+}
+
+/*
+ * The object keys are themselves comma-separated lists of attnums
+ * with negative attnums representing one of the expressions defined
+ * in the extened statistics object.
+ */
+static JsonParseErrorType
+ndistinct_object_field_start(void *state, char *fname, bool isnull)
+{
+ ndistinctParseState *parse = state;
+ char *token;
+ char *saveptr;
+ const char *delim = ", ";
+ char *scratch;
+ List *attnum_list = NIL;
+ int natts = 0;
+ MVNDistinctItem *item;
+
+ if (isnull || fname == NULL)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_ndistinct: \"%s\"", parse->str),
+ errdetail("All ndistinct attnum lists must be a comma separated list of attnums.")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ scratch = pstrdup(fname);
+
+ token = strtok_r(scratch, delim, &saveptr);
+
+ while (token != NULL)
+ {
+ attnum_list = lappend(attnum_list, (void *) token);
+
+ token = strtok_r(NULL, delim, &saveptr);
+ }
+ natts = attnum_list->length;
+
+ /*
+ * We need at least 2 attnums for a ndistinct item, anything less is
+ * malformed.
+ */
+ if (natts < 2)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_ndistinct: \"%s\"", parse->str),
+ errdetail("All ndistinct attnum lists must be a comma separated list of attnums.")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ item = palloc(sizeof(MVNDistinctItem));
+ item->nattributes = natts;
+ item->attributes = palloc(natts * sizeof(AttrNumber));
+
+ for (int i = 0; i < natts; i++)
+ {
+ char *s = (char *) attnum_list->elements[i].ptr_value;
+
+ item->attributes[i] = pg_strtoint16_safe(s, parse->escontext);
+
+ if (SOFT_ERROR_OCCURRED(parse->escontext))
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ list_free(attnum_list);
+ pfree(scratch);
+
+ /* add ndistinct-less MVNDistinctItem to the list */
+ parse->current_item = item;
+ parse->distinct_items = lappend(parse->distinct_items, (void *) item);
+ return JSON_SUCCESS;
+}
+
+/*
+ * ndsitinct input format does not have arrays, so any array elements encountered
+ * are an error.
+ */
+static JsonParseErrorType
+ndistinct_array_element_start(void *state, bool isnull)
+{
+ ndistinctParseState *parse = state;
+
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_ndistinct: \"%s\"", parse->str),
+ errdetail("Cannot contain array elements.")));
+
+ return JSON_SEM_ACTION_FAILED;
+}
+
+/*
+ * Handle scalar events from the ndistinct input parser.
+ *
+ * There is only one case where we will encounter a scalar, and that is the
+ * ndsitinct value for the previous object key.
+ */
+static JsonParseErrorType
+ndistinct_scalar(void *state, char *token, JsonTokenType tokentype)
+{
+ ndistinctParseState *parse = state;
+
+ /* if the entire json is just one scalar, that's wrong */
+ if (parse->found_only_object != true)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_ndistinct: \"%s\"", parse->str),
+ errdetail("Must begin with \"{\"")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ Assert(parse->current_item != NULL);
+
+ parse->current_item->ndistinct = float8in_internal(token, NULL, "double",
+ token, parse->escontext);
+
+ if (SOFT_ERROR_OCCURRED(parse->escontext))
+ return JSON_SEM_ACTION_FAILED;
+
+ /* mark us done with this item */
+ parse->current_item = NULL;
+ return JSON_SUCCESS;
+}
+
/*
* pg_ndistinct_in
* input routine for type pg_ndistinct
*
- * pg_ndistinct is real enough to be a table column, but it has no
- * operations of its own, and disallows input (just like pg_node_tree).
+ * example input: {"6, -1": 14, "6, -2": 9143, "-1, -2": 13454, "6, -1, -2": 14549}
+ *
+ * This import format is clearly a specific subset of JSON, therefore it makes
+ * sense to leverage those parsing utilities, and further validate it from there.
*/
Datum
pg_ndistinct_in(PG_FUNCTION_ARGS)
{
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("cannot accept a value of type %s", "pg_ndistinct")));
+ char *str = PG_GETARG_CSTRING(0);
- PG_RETURN_VOID(); /* keep compiler quiet */
+ ndistinctParseState parse_state;
+ JsonParseErrorType result;
+ JsonLexContext *lex;
+ JsonSemAction sem_action;
+
+ /* initialize semantic state */
+ parse_state.str = str;
+ parse_state.found_only_object = false;
+ parse_state.distinct_items = NIL;
+ parse_state.escontext = fcinfo->context;
+ parse_state.current_item = NULL;
+
+ /* set callbacks */
+ sem_action.semstate = (void *) &parse_state;
+ sem_action.object_start = ndistinct_object_start;
+ sem_action.object_end = NULL;
+ sem_action.array_start = ndistinct_array_start;
+ sem_action.array_end = NULL;
+ sem_action.object_field_start = ndistinct_object_field_start;
+ sem_action.object_field_end = NULL;
+ sem_action.array_element_start = ndistinct_array_element_start;
+ sem_action.array_element_end = NULL;
+ sem_action.scalar = ndistinct_scalar;
+
+ lex = makeJsonLexContextCstringLen(NULL, str, strlen(str),
+ PG_UTF8, true);
+ result = pg_parse_json(lex, &sem_action);
+ freeJsonLexContext(lex);
+ if (result == JSON_SUCCESS)
+ {
+ MVNDistinct *ndistinct;
+ int nitems = parse_state.distinct_items->length;
+ bytea *bytes;
+
+ ndistinct = palloc(offsetof(MVNDistinct, items) +
+ nitems * sizeof(MVNDistinctItem));
+
+ ndistinct->magic = STATS_NDISTINCT_MAGIC;
+ ndistinct->type = STATS_NDISTINCT_TYPE_BASIC;
+ ndistinct->nitems = nitems;
+
+ for (int i = 0; i < nitems; i++)
+ {
+ MVNDistinctItem *item = parse_state.distinct_items->elements[i].ptr_value;
+
+ ndistinct->items[i].ndistinct = item->ndistinct;
+ ndistinct->items[i].nattributes = item->nattributes;
+ ndistinct->items[i].attributes = item->attributes;
+
+ /*
+ * free the MVNDistinctItem, but not the attributes we're still
+ * using
+ */
+ pfree(item);
+ }
+ bytes = statext_ndistinct_serialize(ndistinct);
+
+ list_free(parse_state.distinct_items);
+ for (int i = 0; i < nitems; i++)
+ pfree(ndistinct->items[i].attributes);
+ pfree(ndistinct);
+
+ PG_RETURN_BYTEA_P(bytes);
+ }
+ else if (result == JSON_SEM_ACTION_FAILED)
+ PG_RETURN_NULL(); /* escontext already set */
+
+ /* Anything else is a generic JSON parse error */
+ ereturn(parse_state.escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_ndistinct: \"%s\"", str),
+ errdetail("Must be valid JSON.")));
+ PG_RETURN_NULL();
}
/*
diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out
index a4c7be487e..6f3da85101 100644
--- a/src/test/regress/expected/stats_ext.out
+++ b/src/test/regress/expected/stats_ext.out
@@ -3335,6 +3335,13 @@ SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x
s_expr | {1}
(2 rows)
+-- new input functions
+SELECT '{"6, -1": 14, "6, -2": 9143, "-1, -2": 13454, "6, -1, -2": 14549}'::pg_ndistinct;
+ pg_ndistinct
+-------------------------------------------------------------------
+ {"6, -1": 14, "6, -2": 9143, "-1, -2": 13454, "6, -1, -2": 14549}
+(1 row)
+
-- Tidy up
DROP OPERATOR <<< (int, int);
DROP FUNCTION op_leak(int, int);
diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql
index 5c786b16c6..a53564bed5 100644
--- a/src/test/regress/sql/stats_ext.sql
+++ b/src/test/regress/sql/stats_ext.sql
@@ -1686,6 +1686,9 @@ SELECT statistics_name, most_common_vals FROM pg_stats_ext x
SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x
WHERE tablename = 'stats_ext_tbl' ORDER BY ROW(x.*);
+-- new input functions
+SELECT '{"6, -1": 14, "6, -2": 9143, "-1, -2": 13454, "6, -1, -2": 14549}'::pg_ndistinct;
+
-- Tidy up
DROP OPERATOR <<< (int, int);
DROP FUNCTION op_leak(int, int);
--
2.47.1
[text/x-patch] v37-0003-Add-working-input-function-for-pg_dependencies.patch (12.6K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/5-v37-0003-Add-working-input-function-for-pg_dependencies.patch)
download | inline diff:
From a7128a0fe5a14e09bbd5a832eac4b8914dc717b2 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Tue, 17 Dec 2024 19:47:43 -0500
Subject: [PATCH v37 03/11] Add working input function for pg_dependencies.
This is needed to import extended statistics.
---
src/backend/statistics/dependencies.c | 322 +++++++++++++++++++++++-
src/test/regress/expected/stats_ext.out | 6 +
src/test/regress/sql/stats_ext.sql | 1 +
3 files changed, 319 insertions(+), 10 deletions(-)
diff --git a/src/backend/statistics/dependencies.c b/src/backend/statistics/dependencies.c
index eb2fc4366b..a26f73d063 100644
--- a/src/backend/statistics/dependencies.c
+++ b/src/backend/statistics/dependencies.c
@@ -13,18 +13,26 @@
*/
#include "postgres.h"
+#include "access/attnum.h"
#include "access/htup_details.h"
#include "catalog/pg_statistic_ext.h"
#include "catalog/pg_statistic_ext_data.h"
+#include "common/jsonapi.h"
+#include "fmgr.h"
#include "lib/stringinfo.h"
+#include "mb/pg_wchar.h"
+#include "nodes/miscnodes.h"
#include "nodes/nodeFuncs.h"
#include "nodes/nodes.h"
#include "nodes/pathnodes.h"
+#include "nodes/pg_list.h"
#include "optimizer/clauses.h"
#include "optimizer/optimizer.h"
#include "parser/parsetree.h"
#include "statistics/extended_stats_internal.h"
#include "statistics/statistics.h"
+#include "utils/builtins.h"
+#include "utils/float.h"
#include "utils/fmgroids.h"
#include "utils/fmgrprotos.h"
#include "utils/lsyscache.h"
@@ -643,24 +651,318 @@ statext_dependencies_load(Oid mvoid, bool inh)
return result;
}
+typedef struct
+{
+ const char *str;
+ bool found_only_object;
+ List *dependency_list;
+ Node *escontext;
+
+ MVDependency *current_dependency;
+} dependenciesParseState;
+
+/*
+ * Invoked at the start of each object in the JSON document.
+ * The entire JSON document should be one object with no sub-objects.
+ *
+ * If we're anywhere else in the document, it's an error.
+ */
+static JsonParseErrorType
+dependencies_object_start(void *state)
+{
+ dependenciesParseState *parse = state;
+
+ if (parse->found_only_object == true)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("Must begin with \"{\"")));
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ parse->found_only_object = true;
+ return JSON_SUCCESS;
+}
+
+/*
+ * dependencies input format does not have arrays, so any array elements encountered
+ * are an error.
+ */
+static JsonParseErrorType
+dependencies_array_start(void *state)
+{
+ dependenciesParseState *parse = state;
+
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("All dependencies count values are scalar doubles.")));
+ return JSON_SEM_ACTION_FAILED;
+}
+
+/* TODO COPY START */
+
+
+/*
+ * The object keys are themselves comma-separated lists of attnums
+ * with negative attnums representing one of the expressions defined
+ * in the extened statistics object, followed by a => and a final attnum.
+ *
+ * example: "-1, 2 => -1"
+ */
+static JsonParseErrorType
+dependencies_object_field_start(void *state, char *fname, bool isnull)
+{
+ dependenciesParseState *parse = state;
+ char *token;
+ char *saveptr;
+ const char *delim = ", ";
+ const char *arrow_delim = " => ";
+ char *scratch;
+ char *arrow_p;
+ char *after_arrow_p;
+ List *attnum_list = NIL;
+ int natts = 0;
+ AttrNumber final_attnum;
+ MVDependency *dep;
+
+ if (isnull || fname == NULL)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("All dependencies attnum lists must be a comma separated list of attnums with a final => attnum.")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ scratch = pstrdup(fname);
+
+ /* The subtring ' => ' must occur exactly once */
+ arrow_p = strstr(scratch, arrow_delim);
+ if (arrow_p == NULL)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("All dependencies attnum lists must be a comma separated list of attnums with a final => attnum.")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ /*
+ * Everything to the left of the arrow is the attribute list, so split
+ * that off into its own string.
+ *
+ * Everything to the right should be the lone target attribute.
+ */
+ *arrow_p = '\0';
+
+ /* look for the character immediately beyond the delimiter we just found */
+ after_arrow_p = arrow_p + strlen(arrow_delim);
+
+ /* We should not find another arrow delim */
+ if (strstr(after_arrow_p, arrow_delim) != NULL)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("All dependencies attnum lists must be a comma separated list of attnums with a final => attnum.")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ /* what is left should be exactly one attnum */
+ final_attnum = pg_strtoint16_safe(after_arrow_p, parse->escontext);
+
+ if (SOFT_ERROR_OCCURRED(parse->escontext))
+ return JSON_SEM_ACTION_FAILED;
+
+ /* Left of the arrow is just regular attnums */
+ token = strtok_r(scratch, delim, &saveptr);
+
+ while (token != NULL)
+ {
+ attnum_list = lappend(attnum_list, (void *) token);
+
+ token = strtok_r(NULL, delim, &saveptr);
+ }
+ natts = attnum_list->length;
+
+ /*
+ * We need at least 2 attnums left of the arrow for a dependencies item,
+ * anything less is malformed.
+ */
+ if (natts < 1)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("All dependencies attnum lists must be a comma separated list of attnums.")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ /*
+ * Allocate enough space for the dependency, the attnums in the list, plus
+ * the final attnum
+ */
+ dep = palloc0(offsetof(MVDependency, attributes) + ((natts + 1) * sizeof(AttrNumber)));
+ dep->nattributes = natts + 1;
+ dep->attributes[natts] = final_attnum;
+
+ for (int i = 0; i < natts; i++)
+ {
+ char *s = (char *) attnum_list->elements[i].ptr_value;
+
+ dep->attributes[i] = pg_strtoint16_safe(s, parse->escontext);
+
+ if (SOFT_ERROR_OCCURRED(parse->escontext))
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ list_free(attnum_list);
+ pfree(scratch);
+
+ /* add dependencies-less MVdependenciesItem to the list */
+ parse->current_dependency = dep;
+ parse->dependency_list = lappend(parse->dependency_list, (void *) dep);
+ return JSON_SUCCESS;
+}
+
+/*
+ * ndsitinct input format does not have arrays, so any array elements encountered
+ * are an error.
+ */
+static JsonParseErrorType
+dependencies_array_element_start(void *state, bool isnull)
+{
+ dependenciesParseState *parse = state;
+
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("Cannot contain array elements.")));
+
+ return JSON_SEM_ACTION_FAILED;
+}
+
+/*
+ * Handle scalar events from the dependencies input parser.
+ *
+ * There is only one case where we will encounter a scalar, and that is the
+ * dependency degree for the previous object key.
+ */
+static JsonParseErrorType
+dependencies_scalar(void *state, char *token, JsonTokenType tokentype)
+{
+ dependenciesParseState *parse = state;
+
+ /* if the entire json is just one scalar, that's wrong */
+ if (parse->found_only_object != true)
+ {
+ ereturn(parse->escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", parse->str),
+ errdetail("Must begin with \"{\"")));
+
+ return JSON_SEM_ACTION_FAILED;
+ }
+
+ Assert(parse->current_dependency != NULL);
+
+ parse->current_dependency->degree = float8in_internal(token, NULL, "double",
+ token, parse->escontext);
+
+ if (SOFT_ERROR_OCCURRED(parse->escontext))
+ return JSON_SEM_ACTION_FAILED;
+
+ /* mark us done with this dependency */
+ parse->current_dependency = NULL;
+ return JSON_SUCCESS;
+}
+
/*
* pg_dependencies_in - input routine for type pg_dependencies.
*
- * pg_dependencies is real enough to be a table column, but it has no operations
- * of its own, and disallows input too
+ * example input:
+ * {"-2 => 6": 0.292508,
+ * "-2 => -1": 0.113999,
+ * "6, -2 => -1": 0.348479,
+ * "-1, -2 => 6": 0.839691}
+ *
+ * This import format is clearly a specific subset of JSON, therefore it makes
+ * sense to leverage those parsing utilities, and further validate it from there.
*/
Datum
pg_dependencies_in(PG_FUNCTION_ARGS)
{
- /*
- * pg_node_list stores the data in binary form and parsing text input is
- * not needed, so disallow this.
- */
- ereport(ERROR,
- (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
- errmsg("cannot accept a value of type %s", "pg_dependencies")));
+ char *str = PG_GETARG_CSTRING(0);
- PG_RETURN_VOID(); /* keep compiler quiet */
+ dependenciesParseState parse_state;
+ JsonParseErrorType result;
+ JsonLexContext *lex;
+ JsonSemAction sem_action;
+
+ /* initialize the semantic state */
+ parse_state.str = str;
+ parse_state.found_only_object = false;
+ parse_state.dependency_list = NIL;
+ parse_state.escontext = fcinfo->context;
+ parse_state.current_dependency = NULL;
+
+ /* set callbacks */
+ sem_action.semstate = (void *) &parse_state;
+ sem_action.object_start = dependencies_object_start;
+ sem_action.object_end = NULL;
+ sem_action.array_start = dependencies_array_start;
+ sem_action.array_end = NULL;
+ sem_action.array_element_start = dependencies_array_element_start;
+ sem_action.array_element_end = NULL;
+ sem_action.object_field_start = dependencies_object_field_start;
+ sem_action.object_field_end = NULL;
+ sem_action.scalar = dependencies_scalar;
+
+ lex = makeJsonLexContextCstringLen(NULL, str, strlen(str), PG_UTF8, true);
+
+ result = pg_parse_json(lex, &sem_action);
+ freeJsonLexContext(lex);
+
+ if (result == JSON_SUCCESS)
+ {
+ List *list = parse_state.dependency_list;
+ int ndeps = list->length;
+ MVDependencies *mvdeps;
+ bytea *bytes;
+
+ mvdeps = palloc0(offsetof(MVDependencies, deps) + ndeps * sizeof(MVDependency));
+ mvdeps->magic = STATS_DEPS_MAGIC;
+ mvdeps->type = STATS_DEPS_TYPE_BASIC;
+ mvdeps->ndeps = ndeps;
+
+ /* copy MVDependency structs out of the list into the MVDependencies */
+ for (int i = 0; i < ndeps; i++)
+ mvdeps->deps[i] = list->elements[i].ptr_value;
+ bytes = statext_dependencies_serialize(mvdeps);
+
+ list_free(list);
+ for (int i = 0; i < ndeps; i++)
+ pfree(mvdeps->deps[i]);
+ pfree(mvdeps);
+
+ PG_RETURN_BYTEA_P(bytes);
+ }
+ else if (result == JSON_SEM_ACTION_FAILED)
+ PG_RETURN_NULL();
+
+ /* Anything else is a generic JSON parse error */
+ ereturn(parse_state.escontext, (Datum) 0,
+ (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
+ errmsg("malformed pg_dependencies: \"%s\"", str),
+ errdetail("Must be valid JSON.")));
+
+ PG_RETURN_NULL(); /* keep compiler quiet */
}
/*
diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out
index 6f3da85101..4dda2d8b9c 100644
--- a/src/test/regress/expected/stats_ext.out
+++ b/src/test/regress/expected/stats_ext.out
@@ -3342,6 +3342,12 @@ SELECT '{"6, -1": 14, "6, -2": 9143, "-1, -2": 13454, "6, -1, -2": 14549}'::pg_n
{"6, -1": 14, "6, -2": 9143, "-1, -2": 13454, "6, -1, -2": 14549}
(1 row)
+SELECT '{"-2 => 6": 0.292508, "-2 => -1": 0.113999, "6, -2 => -1": 0.348479, "-1, -2 => 6": 0.839691}'::pg_dependencies;
+ pg_dependencies
+-----------------------------------------------------------------------------------------------
+ {"-2 => 6": 0.292508, "-2 => -1": 0.113999, "6, -2 => -1": 0.348479, "-1, -2 => 6": 0.839691}
+(1 row)
+
-- Tidy up
DROP OPERATOR <<< (int, int);
DROP FUNCTION op_leak(int, int);
diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql
index a53564bed5..26375e6e46 100644
--- a/src/test/regress/sql/stats_ext.sql
+++ b/src/test/regress/sql/stats_ext.sql
@@ -1688,6 +1688,7 @@ SELECT statistics_name, most_common_vals FROM pg_stats_ext_exprs x
-- new input functions
SELECT '{"6, -1": 14, "6, -2": 9143, "-1, -2": 13454, "6, -1, -2": 14549}'::pg_ndistinct;
+SELECT '{"-2 => 6": 0.292508, "-2 => -1": 0.113999, "6, -2 => -1": 0.348479, "-1, -2 => 6": 0.839691}'::pg_dependencies;
-- Tidy up
DROP OPERATOR <<< (int, int);
--
2.47.1
[text/x-patch] v37-0005-Add-extended-statistics-support-functions.patch (75.8K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/6-v37-0005-Add-extended-statistics-support-functions.patch)
download | inline diff:
From 31b3c5a94c9820745924f862f6daa800edadb19f Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Fri, 3 Jan 2025 13:43:29 -0500
Subject: [PATCH v37 05/11] Add extended statistics support functions.
Add pg_set_extended_stats(), pg_clear_extended_stats(), and
pg_restore_extended_stats(). These function closely mirror their
relation and attribute counterparts, but for extended statistics (i.e.
CREATE STATISTICS) objects.
---
src/include/catalog/pg_proc.dat | 23 +
src/backend/catalog/system_functions.sql | 17 +
src/backend/statistics/extended_stats.c | 1237 +++++++++++++++++++-
src/test/regress/expected/stats_import.out | 330 ++++++
src/test/regress/sql/stats_import.sql | 244 ++++
doc/src/sgml/func.sgml | 124 ++
6 files changed, 1973 insertions(+), 2 deletions(-)
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index b37e8a6f88..96ec30cd6e 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -12428,5 +12428,28 @@
proname => 'gist_stratnum_identity', prorettype => 'int2',
proargtypes => 'int2',
prosrc => 'gist_stratnum_identity' },
+{ oid => '9947',
+ descr => 'restore statistics on extended statistics object',
+ proname => 'pg_restore_extended_stats', provolatile => 'v', proisstrict => 'f',
+ provariadic => 'any',
+ proparallel => 'u', prorettype => 'bool',
+ proargtypes => 'any',
+ proargnames => '{kwargs}',
+ proargmodes => '{v}',
+ prosrc => 'pg_restore_extended_stats' },
+{ oid => '9948',
+ descr => 'set statistics on extended statistics object',
+ proname => 'pg_set_extended_stats', provolatile => 'v', proisstrict => 'f',
+ proparallel => 'u', prorettype => 'void',
+ proargtypes => 'regnamespace name bool pg_ndistinct pg_dependencies _text _bool _float8 _float8 _text',
+ proargnames => '{statistics_schemaname,statistics_name,inherited,n_distinct,dependencies,most_common_vals,most_common_val_nulls,most_common_freqs,most_common_base_freqs,exprs}',
+ prosrc => 'pg_set_extended_stats' },
+{ oid => '9949',
+ descr => 'clear statistics on extended statistics object',
+ proname => 'pg_clear_extended_stats', provolatile => 'v', proisstrict => 'f',
+ proparallel => 'u', prorettype => 'void',
+ proargtypes => 'regnamespace name bool',
+ proargnames => '{statistics_schemaname,statistics_name,inherited}',
+ prosrc => 'pg_clear_extended_stats' },
]
diff --git a/src/backend/catalog/system_functions.sql b/src/backend/catalog/system_functions.sql
index 591157b1d1..82901bd8e1 100644
--- a/src/backend/catalog/system_functions.sql
+++ b/src/backend/catalog/system_functions.sql
@@ -668,6 +668,23 @@ LANGUAGE INTERNAL
CALLED ON NULL INPUT VOLATILE
AS 'pg_set_attribute_stats';
+CREATE OR REPLACE FUNCTION
+ pg_set_extended_stats(statistics_schemaname regnamespace,
+ statistics_name name,
+ inherited bool,
+ n_distinct pg_ndistinct DEFAULT NULL,
+ dependencies pg_dependencies DEFAULT NULL,
+ most_common_vals text[] DEFAULT NULL,
+ most_common_val_nulls boolean[] DEFAULT NULL,
+ most_common_freqs double precision[] DEFAULT NULL,
+ most_common_base_freqs double precision[] DEFAULT NULL,
+ exprs text[] DEFAULT NULL)
+RETURNS void
+LANGUAGE INTERNAL
+CALLED ON NULL INPUT VOLATILE
+AS 'pg_set_extended_stats';
+
+
--
-- The default permissions for functions mean that anyone can execute them.
-- A number of functions shouldn't be executable by just anyone, but rather
diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c
index 34dcb535e1..3d949cf552 100644
--- a/src/backend/statistics/extended_stats.c
+++ b/src/backend/statistics/extended_stats.c
@@ -18,22 +18,35 @@
#include "access/detoast.h"
#include "access/genam.h"
+#include "access/heapam.h"
+#include "access/htup.h"
#include "access/htup_details.h"
#include "access/table.h"
+#include "c.h"
#include "catalog/indexing.h"
+#include "catalog/pg_collation.h"
+#include "catalog/pg_statistic_d.h"
#include "catalog/pg_statistic_ext.h"
+#include "catalog/pg_statistic_ext_d.h"
#include "catalog/pg_statistic_ext_data.h"
+#include "catalog/pg_statistic_ext_data_d.h"
+#include "catalog/pg_type_d.h"
#include "commands/defrem.h"
#include "commands/progress.h"
+#include "commands/vacuum.h"
#include "executor/executor.h"
+#include "fmgr.h"
#include "miscadmin.h"
+#include "nodes/miscnodes.h"
#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "parser/parsetree.h"
#include "pgstat.h"
#include "postmaster/autovacuum.h"
#include "statistics/extended_stats_internal.h"
+#include "statistics/stat_utils.h"
#include "statistics/statistics.h"
+#include "storage/lockdefs.h"
#include "utils/acl.h"
#include "utils/array.h"
#include "utils/attoptcache.h"
@@ -42,9 +55,11 @@
#include "utils/fmgroids.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
+#include "utils/palloc.h"
#include "utils/rel.h"
#include "utils/selfuncs.h"
#include "utils/syscache.h"
+#include "utils/typcache.h"
/*
* To avoid consuming too much memory during analysis and/or too much space
@@ -72,6 +87,71 @@ typedef struct StatExtEntry
List *exprs; /* expressions */
} StatExtEntry;
+enum extended_stats_argnum
+{
+ STATSCHEMA_ARG = 0,
+ STATNAME_ARG,
+ INHERITED_ARG,
+ NDISTINCT_ARG,
+ DEPENDENCIES_ARG,
+ MOST_COMMON_VALS_ARG,
+ MOST_COMMON_VAL_NULLS_ARG,
+ MOST_COMMON_FREQS_ARG,
+ MOST_COMMON_BASE_FREQS_ARG,
+ EXPRESSIONS_ARG,
+ NUM_EXTENDED_STATS_ARGS
+};
+
+static struct StatsArgInfo extarginfo[] =
+{
+ [STATSCHEMA_ARG] = {"statistics_schemaname", REGNAMESPACEOID},
+ [STATNAME_ARG] = {"statistics_name", NAMEOID},
+ [INHERITED_ARG] = {"inherited", BOOLOID},
+ [NDISTINCT_ARG] = {"n_distinct", PG_NDISTINCTOID},
+ [DEPENDENCIES_ARG] = {"dependencies", PG_DEPENDENCIESOID},
+ [MOST_COMMON_VALS_ARG] = {"most_common_vals", TEXTARRAYOID},
+ [MOST_COMMON_VAL_NULLS_ARG] = {"most_common_val_nulls", BOOLARRAYOID},
+ [MOST_COMMON_FREQS_ARG] = {"most_common_freqs", FLOAT8ARRAYOID},
+ [MOST_COMMON_BASE_FREQS_ARG] = {"most_common_base_freqs", FLOAT8ARRAYOID},
+ [EXPRESSIONS_ARG] = {"exprs", TEXTARRAYOID},
+ [NUM_EXTENDED_STATS_ARGS] = {0}
+};
+
+/*
+ * NOTE: the RANGE_LENGTH & RANGE_BOUNDS stats are not yet reflected in any
+ * version of pg_stat_ext_exprs.
+ */
+enum extended_stats_exprs_element
+{
+ NULL_FRAC_ELEM = 0,
+ AVG_WIDTH_ELEM,
+ N_DISTINCT_ELEM,
+ MOST_COMMON_VALS_ELEM,
+ MOST_COMMON_FREQS_ELEM,
+ HISTOGRAM_BOUNDS_ELEM,
+ CORRELATION_ELEM,
+ MOST_COMMON_ELEMS_ELEM,
+ MOST_COMMON_ELEM_FREQS_ELEM,
+ ELEM_COUNT_HISTOGRAM_ELEM,
+ NUM_ATTRIBUTE_STATS_ELEMS
+};
+
+static struct StatsArgInfo extexprarginfo[] =
+{
+ [NULL_FRAC_ELEM] = {"null_frac", FLOAT4OID},
+ [AVG_WIDTH_ELEM] = {"avg_width", INT4OID},
+ [N_DISTINCT_ELEM] = {"n_distinct", FLOAT4OID},
+ [MOST_COMMON_VALS_ELEM] = {"most_common_vals", TEXTOID},
+ [MOST_COMMON_FREQS_ELEM] = {"most_common_freqs", FLOAT4ARRAYOID},
+ [HISTOGRAM_BOUNDS_ELEM] = {"histogram_bounds", TEXTOID},
+ [CORRELATION_ELEM] = {"correlation", FLOAT4OID},
+ [MOST_COMMON_ELEMS_ELEM] = {"most_common_elems", TEXTOID},
+ [MOST_COMMON_ELEM_FREQS_ELEM] = {"most_common_elem_freqs", FLOAT4ARRAYOID},
+ [ELEM_COUNT_HISTOGRAM_ELEM] = {"elem_count_histogram", FLOAT4ARRAYOID},
+ [NUM_ATTRIBUTE_STATS_ELEMS] = {0}
+};
+
+static bool extended_statistics_update(FunctionCallInfo fcinfo, int elevel);
static List *fetch_statentries_for_relation(Relation pg_statext, Oid relid);
static VacAttrStats **lookup_var_attr_stats(Bitmapset *attrs, List *exprs,
@@ -99,6 +179,32 @@ static StatsBuildData *make_build_data(Relation rel, StatExtEntry *stat,
int numrows, HeapTuple *rows,
VacAttrStats **stats, int stattarget);
+static HeapTuple get_pg_statistic_ext(Relation pg_stext, Oid nspoid, Name stxname);
+static bool delete_pg_statistic_ext_data(Oid stxoid, bool inherited);
+
+typedef struct
+{
+ bool ndistinct;
+ bool dependencies;
+ bool mcv;
+ bool expressions;
+} stakindFlags;
+
+static void expand_stxkind(HeapTuple tup, stakindFlags * enabled);
+static void upsert_pg_statistic_ext_data(Datum *values, bool *nulls, bool *replaces);
+static bool check_mcvlist_array(ArrayType *arr, int argindex,
+ int required_ndimss, int mcv_length,
+ int elevel);
+static Datum import_mcvlist(HeapTuple tup, int elevel, int numattrs,
+ Oid *atttypids, int32 *atttypmods, Oid *atttypcolls,
+ ArrayType *mcv_arr, ArrayType *nulls_arr,
+ ArrayType *freqs_arr, ArrayType *base_freqs_arr);
+static Datum import_expressions(Relation pgsd, int elevel, int numexprs,
+ Oid *atttypids, int32 *atttypmods,
+ Oid *atttypcolls, ArrayType *exprs_arr);
+static bool text_to_float4(Datum input, Datum *output);
+static bool text_to_int4(Datum input, Datum *output);
+
/*
* Compute requested extended stats, using the rows sampled for the plain
@@ -2099,7 +2205,6 @@ examine_opclause_args(List *args, Node **exprp, Const **cstp,
return true;
}
-
/*
* Compute statistics about expressions of a relation.
*/
@@ -2239,7 +2344,6 @@ compute_expr_stats(Relation onerel, AnlExprData *exprdata, int nexprs,
MemoryContextDelete(expr_context);
}
-
/*
* Fetch function for analyzing statistics object expressions.
*
@@ -2631,3 +2735,1132 @@ make_build_data(Relation rel, StatExtEntry *stat, int numrows, HeapTuple *rows,
return result;
}
+
+static HeapTuple
+get_pg_statistic_ext(Relation pg_stext, Oid nspoid, Name stxname)
+{
+ ScanKeyData key[2];
+ SysScanDesc scan;
+ HeapTuple tup;
+ Oid stxoid = InvalidOid;
+
+ ScanKeyInit(&key[0],
+ Anum_pg_statistic_ext_stxname,
+ BTEqualStrategyNumber,
+ F_NAMEEQ,
+ NameGetDatum(stxname));
+ ScanKeyInit(&key[1],
+ Anum_pg_statistic_ext_stxnamespace,
+ BTEqualStrategyNumber,
+ F_OIDEQ,
+ ObjectIdGetDatum(nspoid));
+
+ /*
+ * Try to find matching pg_statistic_ext row.
+ */
+ scan = systable_beginscan(pg_stext,
+ StatisticExtNameIndexId,
+ true,
+ NULL,
+ 2,
+ key);
+
+ /* Unique index, either we get a tuple or we don't. */
+ tup = systable_getnext(scan);
+
+ if (HeapTupleIsValid(tup))
+ stxoid = ((Form_pg_statistic_ext) GETSTRUCT(tup))->oid;
+
+ systable_endscan(scan);
+
+ if (!OidIsValid(stxoid))
+ return NULL;
+
+ return SearchSysCacheCopy1(STATEXTOID, ObjectIdGetDatum(stxoid));
+}
+
+/*
+ * Decode the stxkind column so that we know which stats types to expect.
+ */
+static void
+expand_stxkind(HeapTuple tup, stakindFlags * enabled)
+{
+ Datum datum;
+ ArrayType *arr;
+ char *kinds;
+
+ datum = SysCacheGetAttrNotNull(STATEXTOID,
+ tup,
+ Anum_pg_statistic_ext_stxkind);
+ arr = DatumGetArrayTypeP(datum);
+ if (ARR_NDIM(arr) != 1 || ARR_HASNULL(arr) || ARR_ELEMTYPE(arr) != CHAROID)
+ elog(ERROR, "stxkind is not a 1-D char array");
+
+ kinds = (char *) ARR_DATA_PTR(arr);
+
+ for (int i = 0; i < ARR_DIMS(arr)[0]; i++)
+ if (kinds[i] == STATS_EXT_NDISTINCT)
+ enabled->ndistinct = true;
+ else if (kinds[i] == STATS_EXT_DEPENDENCIES)
+ enabled->dependencies = true;
+ else if (kinds[i] == STATS_EXT_MCV)
+ enabled->mcv = true;
+ else if (kinds[i] == STATS_EXT_EXPRESSIONS)
+ enabled->expressions = true;
+}
+
+static void
+upsert_pg_statistic_ext_data(Datum *values, bool *nulls, bool *replaces)
+{
+ Relation pg_stextdata;
+ HeapTuple stxdtup;
+ HeapTuple newtup;
+
+ pg_stextdata = table_open(StatisticExtDataRelationId, RowExclusiveLock);
+
+ stxdtup = SearchSysCache2(STATEXTDATASTXOID,
+ values[Anum_pg_statistic_ext_data_stxoid - 1],
+ values[Anum_pg_statistic_ext_data_stxdinherit - 1]);
+
+ if (HeapTupleIsValid(stxdtup))
+ {
+ newtup = heap_modify_tuple(stxdtup,
+ RelationGetDescr(pg_stextdata),
+ values,
+ nulls,
+ replaces);
+ CatalogTupleUpdate(pg_stextdata, &newtup->t_self, newtup);
+ ReleaseSysCache(stxdtup);
+ }
+ else
+ {
+ newtup = heap_form_tuple(RelationGetDescr(pg_stextdata), values, nulls);
+ CatalogTupleInsert(pg_stextdata, newtup);
+ }
+
+ heap_freetuple(newtup);
+
+ CommandCounterIncrement();
+
+ table_close(pg_stextdata, RowExclusiveLock);
+}
+
+/*
+ * Insert or Update Extended Statistics
+ *
+ * Major errors, such as the table not existing, the statistics object not
+ * existing, or a permissions failure are always reported at ERROR. Other
+ * errors, such as a conversion failure on one statistic kind, are reported
+ * at 'elevel', and other statistic kinds may still be updated.
+ */
+static bool
+extended_statistics_update(FunctionCallInfo fcinfo, int elevel)
+{
+ Oid nspoid;
+ Name stxname;
+ bool inherited;
+ Relation pg_stext;
+ HeapTuple tup = NULL;
+
+ stakindFlags enabled;
+ stakindFlags has;
+
+ Form_pg_statistic_ext stxform;
+
+ Datum values[Natts_pg_statistic_ext_data];
+ bool nulls[Natts_pg_statistic_ext_data];
+ bool replaces[Natts_pg_statistic_ext_data];
+
+ bool success = true;
+
+ int numattnums = 0;
+ int numexprs = 0;
+ int numattrs = 0;
+
+ /* arrays of type info, if we need them */
+ Oid *atttypids = NULL;
+ int32 *atttypmods = NULL;
+ Oid *atttypcolls = NULL;
+
+ memset(nulls, false, sizeof(nulls));
+ memset(values, 0, sizeof(values));
+ memset(replaces, 0, sizeof(replaces));
+ memset(&enabled, 0, sizeof(enabled));
+
+ has.mcv = (!PG_ARGISNULL(MOST_COMMON_VALS_ARG) &&
+ !PG_ARGISNULL(MOST_COMMON_VAL_NULLS_ARG) &&
+ !PG_ARGISNULL(MOST_COMMON_FREQS_ARG) &&
+ !PG_ARGISNULL(MOST_COMMON_BASE_FREQS_ARG));
+ has.ndistinct = !PG_ARGISNULL(NDISTINCT_ARG);
+ has.dependencies = !PG_ARGISNULL(DEPENDENCIES_ARG);
+ has.expressions = !PG_ARGISNULL(EXPRESSIONS_ARG);
+
+ if (RecoveryInProgress())
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("recovery is in progress"),
+ errhint("Statistics cannot be modified during recovery.")));
+
+ stats_check_required_arg(fcinfo, extarginfo, STATSCHEMA_ARG);
+ nspoid = PG_GETARG_OID(STATSCHEMA_ARG);
+ stats_check_required_arg(fcinfo, extarginfo, STATNAME_ARG);
+ stxname = PG_GETARG_NAME(STATNAME_ARG);
+ stats_check_required_arg(fcinfo, extarginfo, INHERITED_ARG);
+ inherited = PG_GETARG_NAME(INHERITED_ARG);
+
+ pg_stext = table_open(StatisticExtRelationId, RowExclusiveLock);
+ tup = get_pg_statistic_ext(pg_stext, nspoid, stxname);
+
+ if (!HeapTupleIsValid(tup))
+ {
+ table_close(pg_stext, RowExclusiveLock);
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("Extended Statistics Object \"%s\".\"%s\" not found.",
+ get_namespace_name(nspoid),
+ NameStr(*stxname))));
+ return false;
+ }
+
+ stxform = (Form_pg_statistic_ext) GETSTRUCT(tup);
+ expand_stxkind(tup, &enabled);
+
+ /* lock table */
+ stats_lock_check_privileges(stxform->stxrelid);
+
+ if (has.mcv)
+ {
+ if (!enabled.mcv)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("MCV parameters \"%s\", \"%s\", \"%s\", and \"%s\" were all "
+ "specified for extended statistics object that does not expect MCV ",
+ extarginfo[MOST_COMMON_VALS_ARG].argname,
+ extarginfo[MOST_COMMON_VAL_NULLS_ARG].argname,
+ extarginfo[MOST_COMMON_FREQS_ARG].argname,
+ extarginfo[MOST_COMMON_BASE_FREQS_ARG].argname)));
+ has.mcv = false;
+ success = false;
+ }
+ }
+ else
+ {
+ /* The MCV args must all be NULL */
+ if (!PG_ARGISNULL(MOST_COMMON_VALS_ARG) ||
+ !PG_ARGISNULL(MOST_COMMON_VAL_NULLS_ARG) ||
+ !PG_ARGISNULL(MOST_COMMON_FREQS_ARG) ||
+ !PG_ARGISNULL(MOST_COMMON_BASE_FREQS_ARG))
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("MCV parameters \"%s\", \"%s\", \"%s\", and \"%s\" must be all specified if any are specified",
+ extarginfo[MOST_COMMON_VALS_ARG].argname,
+ extarginfo[MOST_COMMON_VAL_NULLS_ARG].argname,
+ extarginfo[MOST_COMMON_FREQS_ARG].argname,
+ extarginfo[MOST_COMMON_BASE_FREQS_ARG].argname)));
+ }
+
+ if (has.ndistinct && !enabled.ndistinct)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameters \"%s\" was specified for extended statistics object "
+ "that does not expect \"%s\"",
+ extarginfo[NDISTINCT_ARG].argname,
+ extarginfo[NDISTINCT_ARG].argname)));
+ has.ndistinct = false;
+ success = false;
+ }
+
+ if (has.dependencies && !enabled.dependencies)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameters \"%s\" was specified for extended statistics object "
+ "that does not expect \"%s\"",
+ extarginfo[DEPENDENCIES_ARG].argname,
+ extarginfo[DEPENDENCIES_ARG].argname)));
+ has.dependencies = false;
+ success = false;
+ }
+
+ if (has.expressions && !enabled.expressions)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameters \"%s\" was specified for extended statistics object "
+ "that does not expect \"%s\"",
+ extarginfo[DEPENDENCIES_ARG].argname,
+ extarginfo[DEPENDENCIES_ARG].argname)));
+ has.expressions = false;
+ success = false;
+ }
+
+ /*
+ * Either of these statsistic types requires that we supply
+ * semi-filled-out VacAttrStatP array.
+ *
+ *
+ * It is not possible to use the existing lookup_var_attr_stats() and
+ * examine_attribute() because these functions will skip attributes for
+ * which attstattarget is 0, and we may have stats to import for those
+ * attributes.
+ */
+ if (has.mcv || has.expressions)
+ {
+ Datum exprdatum;
+ bool isnull;
+ List *exprs = NIL;
+
+ /* decode expression (if any) */
+ exprdatum = SysCacheGetAttr(STATEXTOID,
+ tup,
+ Anum_pg_statistic_ext_stxexprs,
+ &isnull);
+
+ if (!isnull)
+ {
+ char *s;
+
+ s = TextDatumGetCString(exprdatum);
+ exprs = (List *) stringToNode(s);
+ pfree(s);
+
+ /*
+ * Run the expressions through eval_const_expressions. This is not
+ * just an optimization, but is necessary, because the planner
+ * will be comparing them to similarly-processed qual clauses, and
+ * may fail to detect valid matches without this. We must not use
+ * canonicalize_qual, however, since these aren't qual
+ * expressions.
+ */
+ exprs = (List *) eval_const_expressions(NULL, (Node *) exprs);
+
+ /* May as well fix opfuncids too */
+ fix_opfuncids((Node *) exprs);
+ }
+
+ numattnums = stxform->stxkeys.dim1;
+ numexprs = list_length(exprs);
+ numattrs = numattnums + numexprs;
+
+ atttypids = palloc0(numattrs * sizeof(Oid));
+ atttypmods = palloc0(numattrs * sizeof(int32));
+ atttypcolls = palloc0(numattrs * sizeof(Oid));
+
+ for (int i = 0; i < numattnums; i++)
+ {
+ AttrNumber attnum = stxform->stxkeys.values[i];
+
+ Oid lt_opr;
+ Oid eq_opr;
+ char typetype;
+
+ /*
+ * fetch attribute entries the same as are done for attribute
+ * stats
+ */
+ get_attr_stat_type(stxform->stxrelid,
+ attnum,
+ elevel,
+ &atttypids[i],
+ &atttypmods[i],
+ &typetype,
+ &atttypcolls[i],
+ <_opr,
+ &eq_opr);
+ }
+
+ for (int i = numattnums; i < numattrs; i++)
+ {
+ Node *expr = list_nth(exprs, i - numattnums);
+
+ atttypids[i] = exprType(expr);
+ atttypmods[i] = exprTypmod(expr);
+ atttypcolls[i] = exprCollation(expr);
+
+ /*
+ * Duplicate logic from get_attr_stat_type
+ */
+
+ /*
+ * If it's a multirange, step down to the range type, as is done by
+ * multirange_typanalyze().
+ */
+ if (type_is_multirange(atttypids[i]))
+ atttypids[i] = get_multirange_range(atttypids[i]);
+
+ /*
+ * Special case: collation for tsvector is DEFAULT_COLLATION_OID. See
+ * compute_tsvector_stats().
+ */
+ if (atttypids[i] == TSVECTOROID)
+ atttypcolls[i] = DEFAULT_COLLATION_OID;
+
+ }
+ }
+
+ /* Primary Key: cannot be NULL or replaced. */
+ values[Anum_pg_statistic_ext_data_stxoid - 1] = ObjectIdGetDatum(stxform->oid);
+ values[Anum_pg_statistic_ext_data_stxdinherit - 1] = BoolGetDatum(inherited);
+
+ if (has.ndistinct)
+ {
+ values[Anum_pg_statistic_ext_data_stxdndistinct - 1] = PG_GETARG_DATUM(NDISTINCT_ARG);
+ replaces[Anum_pg_statistic_ext_data_stxdndistinct - 1] = true;
+ }
+ else
+ nulls[Anum_pg_statistic_ext_data_stxdndistinct - 1] = true;
+
+ if (has.dependencies)
+ {
+ values[Anum_pg_statistic_ext_data_stxddependencies - 1] = PG_GETARG_DATUM(DEPENDENCIES_ARG);
+ replaces[Anum_pg_statistic_ext_data_stxddependencies - 1] = true;
+ }
+ else
+ nulls[Anum_pg_statistic_ext_data_stxddependencies - 1] = true;
+
+ if (has.mcv)
+ {
+ Datum datum;
+
+ datum = import_mcvlist(tup, elevel, numattrs,
+ atttypids, atttypmods, atttypcolls,
+ PG_GETARG_ARRAYTYPE_P(MOST_COMMON_VALS_ARG),
+ PG_GETARG_ARRAYTYPE_P(MOST_COMMON_VAL_NULLS_ARG),
+ PG_GETARG_ARRAYTYPE_P(MOST_COMMON_FREQS_ARG),
+ PG_GETARG_ARRAYTYPE_P(MOST_COMMON_BASE_FREQS_ARG));
+
+ values[Anum_pg_statistic_ext_data_stxdmcv - 1] = datum;
+ replaces[Anum_pg_statistic_ext_data_stxdmcv - 1] = true;
+ }
+ else
+ nulls[Anum_pg_statistic_ext_data_stxdmcv - 1] = true;
+
+ if (has.expressions)
+ {
+ Datum datum;
+ Relation pgsd;
+
+ pgsd = table_open(StatisticRelationId, RowExclusiveLock);
+
+ datum = import_expressions(pgsd, elevel, numexprs,
+ &atttypids[numattnums], &atttypmods[numattnums],
+ &atttypcolls[numattnums],
+ PG_GETARG_ARRAYTYPE_P(EXPRESSIONS_ARG));
+
+ table_close(pgsd, RowExclusiveLock);
+
+ values[Anum_pg_statistic_ext_data_stxdexpr - 1] = datum;
+ replaces[Anum_pg_statistic_ext_data_stxdexpr - 1] = true;
+ }
+ else
+ nulls[Anum_pg_statistic_ext_data_stxdexpr - 1] = true;
+
+ upsert_pg_statistic_ext_data(values, nulls, replaces);
+
+ heap_freetuple(tup);
+ table_close(pg_stext, RowExclusiveLock);
+
+ if (atttypids != NULL)
+ pfree(atttypids);
+ if (atttypmods != NULL)
+ pfree(atttypmods);
+ if (atttypcolls != NULL)
+ pfree(atttypcolls);
+ return success;
+}
+
+ /*
+ * The MCV is an array of records, but this is expected as 4 separate arrays.
+ * It is not possible to have a generic input function for pg_mcv_list
+ * because the most_common_values is a composite type with element types
+ * defined by the specific statistics object.
+ */
+static Datum
+import_mcvlist(HeapTuple tup, int elevel, int numattrs, Oid *atttypids,
+ int32 *atttypmods, Oid *atttypcolls,
+ ArrayType *mcv_arr, ArrayType *nulls_arr, ArrayType *freqs_arr,
+ ArrayType *base_freqs_arr)
+{
+ int nitems;
+
+ MCVList *mcvlist;
+ bytea *bytes;
+
+ Datum *mcv_elems;
+ bool *mcv_nulls;
+ int check_nummcv;
+
+ bool *mcv_elem_nulls;
+ float8 *freqs;
+ float8 *base_freqs;
+
+ HeapTuple *vatuples;
+ VacAttrStats **vastats;
+
+ /*
+ * The mcv_arr is an array of arrays of text, and we use it as the reference
+ * array for checking the lengths of the other 3 arrays.
+ */
+ if (ARR_NDIM(mcv_arr) != 2)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameters \"%s\" must be a text array of 2 dimensions.",
+ extarginfo[MOST_COMMON_VALS_ARG].argname)));
+ return (Datum) 0;
+ }
+
+ nitems = ARR_DIMS(mcv_arr)[0];
+
+ /* fixed length arrays that cannot contain NULLs */
+ if (!check_mcvlist_array(nulls_arr, MOST_COMMON_VAL_NULLS_ARG,
+ 2, nitems, elevel) ||
+ !check_mcvlist_array(freqs_arr, MOST_COMMON_FREQS_ARG,
+ 1, nitems, elevel ) ||
+ !check_mcvlist_array(base_freqs_arr, MOST_COMMON_BASE_FREQS_ARG,
+ 1, nitems, elevel ))
+ return (Datum) 0;
+
+ mcv_elem_nulls = (bool *) ARR_DATA_PTR(nulls_arr);
+ freqs = (float8 *) ARR_DATA_PTR(freqs_arr);
+ base_freqs = (float8 *) ARR_DATA_PTR(base_freqs_arr);
+
+ /*
+ * Allocate the MCV list structure, set the global parameters.
+ */
+ mcvlist = (MCVList *) palloc0(offsetof(MCVList, items) +
+ (sizeof(MCVItem) * nitems));
+
+ mcvlist->magic = STATS_MCV_MAGIC;
+ mcvlist->type = STATS_MCV_TYPE_BASIC;
+ mcvlist->ndimensions = numattrs;
+ mcvlist->nitems = nitems;
+
+ deconstruct_array_builtin(mcv_arr, TEXTOID, &mcv_elems,
+ &mcv_nulls, &check_nummcv);
+
+ Assert(check_nummcv == (nitems*numattrs));
+
+ /* Set the values for the 1-D arrays and allocate space for the 2-D arrays */
+ for (int i = 0; i < nitems; i++)
+ {
+ MCVItem *item = &mcvlist->items[i];
+
+ item->frequency = freqs[i];
+ item->base_frequency = base_freqs[i];
+ item->values = (Datum *) palloc0(sizeof(Datum) * numattrs);
+ item->isnull = (bool *) palloc0(sizeof(bool) * numattrs);
+ }
+
+ /* Walk through each dimension */
+ for (int j = 0; j < numattrs; j++)
+ {
+ FmgrInfo finfo;
+ Oid ioparam;
+ Oid infunc;
+ int index = j;
+
+ getTypeInputInfo(atttypids[j], &infunc, &ioparam);
+ fmgr_info(infunc, &finfo);
+
+ /* store info about data type OIDs */
+ mcvlist->types[j] = atttypids[j];
+
+ for (int i = 0; i < nitems; i++)
+ {
+ MCVItem *item = &mcvlist->items[i];
+
+ /* These should be in agreement, but just to be safe check both */
+ if (mcv_elem_nulls[index] || mcv_nulls[index])
+ {
+ item->values[j] = (Datum) 0;
+ item->isnull[j] = true;
+ }
+ else
+ {
+ char *s = TextDatumGetCString(mcv_elems[index]);
+ ErrorSaveContext escontext = {T_ErrorSaveContext};
+
+ if (!InputFunctionCallSafe(&finfo, s, ioparam, atttypmods[j],
+ (fmNodePtr) &escontext, &item->values[j]))
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("MCV elemement \"%s\" does not match expected input type.", s)));
+ return (Datum) 0;
+ }
+
+ pfree(s);
+ }
+
+ index += numattrs;
+ }
+ }
+
+ /*
+ * The function statext_mcv_serialize() requires an array of pointers
+ * to VacAttrStats records, but only a few fields within those records
+ * have to be filled out.
+ */
+ vastats = (VacAttrStats **) palloc0(numattrs * sizeof(VacAttrStats));
+ vatuples = (HeapTuple *) palloc0(numattrs * sizeof(HeapTuple));
+
+ for (int i = 0; i < numattrs; i++)
+ {
+ Oid typid = atttypids[i];
+ HeapTuple typtuple;
+
+ typtuple = SearchSysCacheCopy1(TYPEOID, ObjectIdGetDatum(typid));
+
+ if (!HeapTupleIsValid(typtuple))
+ elog(ERROR, "cache lookup failed for type %u", typid);
+
+ vatuples[i] = typtuple;
+
+ vastats[i] = palloc0(sizeof(VacAttrStats));
+
+ vastats[i]->attrtype = (Form_pg_type) GETSTRUCT(typtuple);
+ vastats[i]->attrtypid = typid;
+ vastats[i]->attrcollid = atttypcolls[i];
+ }
+
+ bytes = statext_mcv_serialize(mcvlist, vastats);
+
+ for (int i = 0; i < numattrs; i++)
+ {
+ pfree(vatuples[i]);
+ pfree(vastats[i]);
+ }
+ pfree((void *) vatuples);
+ pfree((void *) vastats);
+
+ if (bytes == NULL)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Unable to import mcv list")));
+ return (Datum) 0;
+ }
+
+ for (int i = 0; i < nitems; i++)
+ {
+ MCVItem *item = &mcvlist->items[i];
+
+ pfree(item->values);
+ pfree(item->isnull);
+ }
+ pfree(mcvlist);
+ pfree(mcv_elems);
+ pfree(mcv_nulls);
+
+ return PointerGetDatum(bytes);
+}
+
+/*
+ * Consistency checks to ensure that other mcvlist arrays are in alignment
+ * with the mcv array.
+ */
+static
+bool check_mcvlist_array(ArrayType *arr, int argindex, int required_ndims,
+ int mcv_length, int elevel)
+{
+ if (ARR_NDIM(arr) != required_ndims)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameter \"%s\" must be an array of %d dimensions.",
+ extarginfo[argindex].argname, required_ndims)));
+ return false;
+ }
+
+ if (array_contains_nulls(arr))
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Array \"%s\" cannot contain NULLs.",
+ extarginfo[argindex].argname)));
+ return false;
+ }
+
+ if (ARR_DIMS(arr)[0] != mcv_length)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameters \"%s\" must have the same number of elements as \"%s\"",
+ extarginfo[argindex].argname,
+ extarginfo[MOST_COMMON_VALS_ARG].argname)));
+ return false;
+ }
+
+ return true;
+}
+
+/*
+ * Create the stxdexprs datum using the user input in an array of array of
+ * text, referenced against the datatypes for the expressions.
+ */
+static Datum
+import_expressions(Relation pgsd, int elevel, int numexprs,
+ Oid *atttypids, int32 *atttypmods,
+ Oid *atttypcolls, ArrayType *exprs_arr)
+{
+ Datum *exprs_elems;
+ bool *exprs_nulls;
+ int check_numexprs;
+ int offset = 0;
+
+ FmgrInfo array_in_fn;
+
+ Oid pgstypoid = get_rel_type_id(StatisticRelationId);
+
+ ArrayBuildState *astate = NULL;
+
+
+ if (ARR_NDIM(exprs_arr) != 2)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameter \"%s\" must be a text array of 2 dimensions.",
+ extarginfo[EXPRESSIONS_ARG].argname)));
+ return (Datum) 0;
+ }
+
+ if (ARR_DIMS(exprs_arr)[0] != numexprs)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameter \"%s\" must have an outer dimension of %d elements.",
+ extarginfo[EXPRESSIONS_ARG].argname, numexprs)));
+ return (Datum) 0;
+ }
+ if (ARR_DIMS(exprs_arr)[1] != NUM_ATTRIBUTE_STATS_ELEMS)
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Parameter \"%s\" must have an inner dimension of %d elements.",
+ extarginfo[EXPRESSIONS_ARG].argname,
+ NUM_ATTRIBUTE_STATS_ELEMS)));
+ return (Datum) 0;
+ }
+
+ fmgr_info(F_ARRAY_IN, &array_in_fn);
+
+ deconstruct_array_builtin(exprs_arr, TEXTOID, &exprs_elems,
+ &exprs_nulls, &check_numexprs);
+
+ for (int i = 0; i < numexprs; i++)
+ {
+ Oid typid = atttypids[i];
+ int32 typmod = atttypmods[i];
+ Oid stacoll = atttypcolls[i];
+ TypeCacheEntry *typcache;
+
+ Oid elemtypid = InvalidOid;
+ Oid elem_eq_opr = InvalidOid;
+
+ bool ok;
+
+ Datum values[Natts_pg_statistic];
+ bool nulls[Natts_pg_statistic];
+ bool replaces[Natts_pg_statistic];
+
+ HeapTuple pgstup;
+ Datum pgstdat;
+
+ /* finds the right operators even if atttypid is a domain */
+ typcache = lookup_type_cache(typid, TYPECACHE_LT_OPR | TYPECACHE_EQ_OPR);
+
+ init_empty_stats_tuple(InvalidOid, InvalidAttrNumber, false,
+ values, nulls, replaces);
+
+ if (!exprs_nulls[offset + NULL_FRAC_ELEM])
+ {
+ ok = text_to_float4(exprs_elems[offset + NULL_FRAC_ELEM],
+ &values[Anum_pg_statistic_stanullfrac - 1]);
+
+ if (!ok)
+ {
+ char *s = TextDatumGetCString(exprs_elems[offset + NULL_FRAC_ELEM]);
+
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Expression %s element \"%s\" does not match expected input type.",
+ extexprarginfo[NULL_FRAC_ELEM].argname, s)));
+ pfree(s);
+ return (Datum) 0;
+ }
+ }
+
+ if (!exprs_nulls[offset + AVG_WIDTH_ELEM])
+ {
+ ok = text_to_int4(exprs_elems[offset + AVG_WIDTH_ELEM],
+ &values[Anum_pg_statistic_stawidth -1 ]);
+
+ if (!ok)
+ {
+ char *s = TextDatumGetCString(exprs_elems[offset + NULL_FRAC_ELEM]);
+
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Expression %s element \"%s\" does not match expected input type.",
+ extexprarginfo[AVG_WIDTH_ELEM].argname, s)));
+ pfree(s);
+ return (Datum) 0;
+ }
+ }
+
+ if (!exprs_nulls[offset + N_DISTINCT_ELEM])
+ {
+ ok = text_to_float4(exprs_elems[offset + N_DISTINCT_ELEM],
+ &values[Anum_pg_statistic_stadistinct - 1]);
+
+ if (!ok)
+ {
+ char *s = TextDatumGetCString(exprs_elems[offset + NULL_FRAC_ELEM]);
+
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Expression %s element \"%s\" does not match expected input type.",
+ extexprarginfo[N_DISTINCT_ELEM].argname, s)));
+ pfree(s);
+ return (Datum) 0;
+ }
+ }
+
+ /*
+ * The STAKIND statistics are the same as the ones found in attribute stats.
+ * However, these are all derived from text columns, whereas the ones
+ * derived for attribute stats are a mix of datatypes. This limits the
+ * opportunities for code sharing between the two.
+ */
+
+ /* STATISTIC_KIND_MCV */
+ if (exprs_nulls[offset + MOST_COMMON_VALS_ELEM] !=
+ exprs_nulls[offset + MOST_COMMON_FREQS_ELEM])
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Expression %s and %s must both be NOT NULL or both NULL.",
+ extexprarginfo[MOST_COMMON_VALS_ELEM].argname,
+ extexprarginfo[MOST_COMMON_FREQS_ELEM].argname)));
+ return (Datum) 0;
+ }
+
+ if (!exprs_nulls[offset + MOST_COMMON_VALS_ELEM])
+ {
+ Datum stavalues;
+ Datum stanumbers;
+
+ stavalues = text_to_stavalues(extexprarginfo[MOST_COMMON_VALS_ELEM].argname,
+ &array_in_fn, exprs_elems[offset + MOST_COMMON_VALS_ELEM],
+ typid, typmod, elevel, &ok);
+
+ if (!ok)
+ return (Datum) 0;
+
+ stanumbers = text_to_stavalues(extexprarginfo[MOST_COMMON_VALS_ELEM].argname,
+ &array_in_fn, exprs_elems[offset + MOST_COMMON_FREQS_ELEM],
+ FLOAT4OID, -1, elevel, &ok);
+
+ if (!ok)
+ return (Datum) 0;
+
+ set_stats_slot(values, nulls, replaces,
+ STATISTIC_KIND_MCV,
+ typcache->eq_opr, stacoll,
+ stanumbers, false, stavalues, false);
+ }
+
+ /* STATISTIC_KIND_HISTOGRAM */
+ if (!exprs_nulls[offset + HISTOGRAM_BOUNDS_ELEM])
+ {
+ Datum stavalues;
+
+ stavalues = text_to_stavalues(extexprarginfo[HISTOGRAM_BOUNDS_ELEM].argname,
+ &array_in_fn, exprs_elems[offset + HISTOGRAM_BOUNDS_ELEM],
+ typid, typmod, elevel, &ok);
+
+ if (!ok)
+ return (Datum) 0;
+
+ set_stats_slot(values, nulls, replaces,
+ STATISTIC_KIND_HISTOGRAM,
+ typcache->lt_opr, stacoll,
+ 0, true, stavalues, false);
+ }
+
+ /* STATISTIC_KIND_CORRELATION */
+ if (!exprs_nulls[offset + CORRELATION_ELEM])
+ {
+ Datum corr[] = {(Datum) 0};
+ ArrayType *arry;
+ Datum stanumbers;
+
+ ok = text_to_float4(exprs_elems[offset + CORRELATION_ELEM], &corr[0]);
+
+ if (!ok)
+ {
+ char *s = TextDatumGetCString(exprs_elems[offset + CORRELATION_ELEM]);
+
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Expression %s element \"%s\" does not match expected input type.",
+ extexprarginfo[CORRELATION_ELEM].argname, s)));
+ return (Datum) 0;
+ }
+
+ arry = construct_array_builtin(corr, 1, FLOAT4OID);
+
+ stanumbers = PointerGetDatum(arry);
+
+ set_stats_slot(values, nulls, replaces,
+ STATISTIC_KIND_CORRELATION,
+ typcache->lt_opr, stacoll,
+ stanumbers, false, 0, true);
+ }
+
+ /* STATISTIC_KIND_MCELEM */
+ if (exprs_nulls[offset + MOST_COMMON_ELEMS_ELEM] !=
+ exprs_nulls[offset + MOST_COMMON_ELEM_FREQS_ELEM])
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("Expression %s and %s must both be NOT NULL or both NULL.",
+ extexprarginfo[MOST_COMMON_ELEMS_ELEM].argname,
+ extexprarginfo[MOST_COMMON_ELEM_FREQS_ELEM].argname)));
+ return (Datum) 0;
+ }
+
+ /*
+ * We only need to fetch element type and eq operator if we have a stat of
+ * type MCELEM or DECHIST.
+ */
+ if (!exprs_nulls[offset + MOST_COMMON_ELEMS_ELEM] ||
+ !exprs_nulls[offset + ELEM_COUNT_HISTOGRAM_ELEM])
+ {
+ if (!get_elem_stat_type(typid, typcache->typtype,
+ elevel, &elemtypid, &elem_eq_opr))
+ {
+ ereport(elevel,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ (errmsg("unable to determine element type of expression"))));
+ return (Datum) 0;
+ }
+ }
+
+ if (!exprs_nulls[offset + MOST_COMMON_ELEMS_ELEM])
+ {
+ Datum stavalues;
+ Datum stanumbers;
+
+ stavalues = text_to_stavalues(extexprarginfo[MOST_COMMON_ELEMS_ELEM].argname,
+ &array_in_fn,
+ exprs_elems[offset + MOST_COMMON_ELEMS_ELEM],
+ elemtypid, typmod,
+ elevel, &ok);
+
+ if (!ok)
+ return (Datum) 0;
+
+ stanumbers = text_to_stavalues(extexprarginfo[MOST_COMMON_ELEM_FREQS_ELEM].argname,
+ &array_in_fn,
+ exprs_elems[offset + MOST_COMMON_ELEM_FREQS_ELEM],
+ FLOAT4OID, -1, elevel, &ok);
+
+ if (!ok)
+ return (Datum) 0;
+
+ set_stats_slot(values, nulls, replaces,
+ STATISTIC_KIND_MCELEM,
+ elem_eq_opr, stacoll,
+ stanumbers, false, stavalues, false);
+ }
+
+ if (!exprs_nulls[offset + ELEM_COUNT_HISTOGRAM_ELEM])
+ {
+ Datum stanumbers;
+
+ stanumbers = text_to_stavalues(extexprarginfo[ELEM_COUNT_HISTOGRAM_ELEM].argname,
+ &array_in_fn,
+ exprs_elems[offset + ELEM_COUNT_HISTOGRAM_ELEM],
+ FLOAT4OID, -1, elevel, &ok);
+
+ if (!ok)
+ return (Datum) 0;
+
+ set_stats_slot(values, nulls, replaces, STATISTIC_KIND_DECHIST,
+ elem_eq_opr, stacoll,
+ stanumbers, false, 0, true);
+ }
+
+ /*
+ * Currently there are no extended stats exports of the statistic kinds
+ * STATISTIC_KIND_BOUNDS_HISTOGRAM or STATISTIC_KIND_RANGE_LENGTH_HISTOGRAM
+ * so these cannot be imported. These may be added in the future.
+ */
+
+ pgstup = heap_form_tuple(RelationGetDescr(pgsd), values, nulls);
+ pgstdat = heap_copy_tuple_as_datum(pgstup, RelationGetDescr(pgsd));
+ astate = accumArrayResult(astate, pgstdat, false, pgstypoid,
+ CurrentMemoryContext);
+
+ offset += NUM_ATTRIBUTE_STATS_ELEMS;
+ }
+
+ pfree(exprs_elems);
+ pfree(exprs_nulls);
+
+ return makeArrayResult(astate, CurrentMemoryContext);
+}
+
+static
+bool text_to_float4(Datum input, Datum *output)
+{
+ ErrorSaveContext escontext = {T_ErrorSaveContext};
+
+ char *s;
+ bool ok;
+
+ s = TextDatumGetCString(input);
+ ok = DirectInputFunctionCallSafe(float4in, s, InvalidOid, -1,
+ (Node *) &escontext, output);
+
+ pfree(s);
+ return ok;
+}
+
+
+static
+bool text_to_int4(Datum input, Datum *output)
+{
+ ErrorSaveContext escontext = {T_ErrorSaveContext};
+
+ char *s;
+ bool ok;
+
+ s = TextDatumGetCString(input);
+ ok = DirectInputFunctionCallSafe(int4in, s, InvalidOid, -1,
+ (Node *) &escontext, output);
+
+ pfree(s);
+ return ok;
+}
+
+static
+bool delete_pg_statistic_ext_data(Oid stxoid, bool inherited)
+{
+ Relation sed = table_open(StatisticExtDataRelationId, RowExclusiveLock);
+ HeapTuple oldtup;
+ bool result = false;
+
+ /* Is there already a pg_statistic tuple for this attribute? */
+ oldtup = SearchSysCache2(STATEXTDATASTXOID,
+ ObjectIdGetDatum(stxoid),
+ BoolGetDatum(inherited));
+
+ if (HeapTupleIsValid(oldtup))
+ {
+ CatalogTupleDelete(sed, &oldtup->t_self);
+ ReleaseSysCache(oldtup);
+ result = true;
+ }
+
+ table_close(sed, RowExclusiveLock);
+
+ CommandCounterIncrement();
+
+ return result;
+}
+
+ /*
+ * Import statistics for a given statistics object.
+ *
+ * Inserts or replaces a row in pg_statistic_ext_data for the given relation
+ * and statistic object schema+name. It takes input parameters that
+ * correspond to columns in the view pg_stats_ext and pg_stats_ext_exprs.
+ *
+ * Parameters are only superficially validated. Omitting a parameter or
+ * passing NULL leaves the statistic unchanged.
+ *
+ */
+Datum
+pg_set_extended_stats(PG_FUNCTION_ARGS)
+{
+ extended_statistics_update(fcinfo, ERROR);
+ PG_RETURN_VOID();
+}
+
+
+Datum
+pg_restore_extended_stats(PG_FUNCTION_ARGS)
+{
+ LOCAL_FCINFO(positional_fcinfo, NUM_EXTENDED_STATS_ARGS);
+ bool result = true;
+
+ InitFunctionCallInfoData(*positional_fcinfo, NULL, NUM_EXTENDED_STATS_ARGS,
+ InvalidOid, NULL, NULL);
+
+ if (!stats_fill_fcinfo_from_arg_pairs(fcinfo, positional_fcinfo,
+ extarginfo, WARNING))
+ result = false;
+
+ if (!extended_statistics_update(positional_fcinfo, WARNING))
+ result = false;
+
+ PG_RETURN_BOOL(result);
+}
+
+/*
+ * Delete statistics for the given statistics object.
+ */
+Datum
+pg_clear_extended_stats(PG_FUNCTION_ARGS)
+{
+ Oid nspoid;
+ Name stxname;
+ bool inherited;
+ Relation pg_stext;
+ HeapTuple tup;
+
+ Form_pg_statistic_ext stxform;
+
+
+ stats_check_required_arg(fcinfo, extarginfo, STATSCHEMA_ARG);
+ nspoid = PG_GETARG_OID(STATSCHEMA_ARG);
+ stats_check_required_arg(fcinfo, extarginfo, STATNAME_ARG);
+ stxname = PG_GETARG_NAME(STATNAME_ARG);
+ stats_check_required_arg(fcinfo, extarginfo, INHERITED_ARG);
+ inherited = PG_GETARG_NAME(INHERITED_ARG);
+
+ if (RecoveryInProgress())
+ ereport(ERROR,
+ (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+ errmsg("recovery is in progress"),
+ errhint("Statistics cannot be modified during recovery.")));
+
+ pg_stext = table_open(StatisticExtRelationId, RowExclusiveLock);
+ tup = get_pg_statistic_ext(pg_stext, nspoid, stxname);
+
+ if (!HeapTupleIsValid(tup))
+ {
+ table_close(pg_stext, RowExclusiveLock);
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("Extended Statistics Object \"%s\".\"%s\" not found.",
+ get_namespace_name(nspoid),
+ NameStr(*stxname))));
+ return false;
+ }
+
+ stxform = (Form_pg_statistic_ext) GETSTRUCT(tup);
+
+ stats_lock_check_privileges(stxform->stxrelid);
+
+ delete_pg_statistic_ext_data(stxform->oid, inherited);
+ heap_freetuple(tup);
+ table_close(pg_stext, RowExclusiveLock);
+
+ PG_RETURN_VOID();
+}
diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out
index fb50da1cd8..9f3b45fc55 100644
--- a/src/test/regress/expected/stats_import.out
+++ b/src/test/regress/expected/stats_import.out
@@ -1414,11 +1414,15 @@ SELECT 3, 'tre', (3, 3.3, 'TRE', '2003-03-03', NULL)::stats_import.complex_type,
UNION ALL
SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+CREATE STATISTICS stats_import.test_stat ON name, comp, lower(arange), array_length(tags,1)
+FROM stats_import.test;
-- Generate statistics on table with data
ANALYZE stats_import.test;
CREATE TABLE stats_import.test_clone ( LIKE stats_import.test )
WITH (autovacuum_enabled = false);
CREATE INDEX is_odd_clone ON stats_import.test_clone(((comp).a % 2 = 1));
+CREATE STATISTICS stats_import.test_stat_clone ON name, comp, lower(arange), array_length(tags,1)
+FROM stats_import.test_clone;
--
-- Copy stats from test to test_clone, and is_odd to is_odd_clone
--
@@ -1801,6 +1805,332 @@ WHERE s.starelid = 'stats_import.is_odd'::regclass;
---------+------------+-------------+----------+-------------+----------+----------+----------+----------+----------+--------+--------+--------+--------+--------+----------+----------+----------+----------+----------+-------------+-------------+-------------+-------------+-------------+-----+-----+-----+-----+-----+-----------
(0 rows)
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ n_distinct => '{"2, 3": 4, "2, -1": 4, "2, -2": 4, "3, -1": 4, "3, -2": 4, "-1, -2": 3, "2, 3, -1": 4, "2, 3, -2": 4, "2, -1, -2": 4, "3, -1, -2": 4, "2, 3, -1, -2": 4}'::pg_ndistinct
+ );
+ pg_set_extended_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ e.n_distinct, e.dependencies, e.most_common_vals, e.most_common_val_nulls,
+ e.most_common_freqs, e.most_common_base_freqs
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false
+\gx
+-[ RECORD 1 ]----------+----------------------------------------------------------------------------------------------------------------------------------------------------------
+n_distinct | {"2, 3": 4, "2, -1": 4, "2, -2": 4, "3, -1": 4, "3, -2": 4, "-1, -2": 3, "2, 3, -1": 4, "2, 3, -2": 4, "2, -1, -2": 4, "3, -1, -2": 4, "2, 3, -1, -2": 4}
+dependencies |
+most_common_vals |
+most_common_val_nulls |
+most_common_freqs |
+most_common_base_freqs |
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ dependencies => '{"2 => 3": 1.000000, "2 => -1": 1.000000, "2 => -2": 1.000000, "3 => 2": 1.000000, "3 => -1": 1.000000, "3 => -2": 1.000000, "-1 => 2": 0.500000, "-1 => 3": 0.500000, "-1 => -2": 1.000000, "-2 => 2": 0.500000, "-2 => 3": 0.500000, "-2 => -1": 1.000000, "2, 3 => -1": 1.000000, "2, 3 => -2": 1.000000, "2, -1 => 3": 1.000000, "2, -1 => -2": 1.000000, "2, -2 => 3": 1.000000, "2, -2 => -1": 1.000000, "3, -1 => 2": 1.000000, "3, -1 => -2": 1.000000, "3, -2 => 2": 1.000000, "3, -2 => -1": 1.000000, "-1, -2 => 2": 0.500000, "-1, -2 => 3": 0.500000, "2, 3, -1 => -2": 1.000000, "2, 3, -2 => -1": 1.000000, "2, -1, -2 => 3": 1.000000, "3, -1, -2 => 2": 1.000000}'::pg_dependencies
+ );
+ pg_set_extended_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ e.n_distinct, e.dependencies, e.most_common_vals, e.most_common_val_nulls,
+ e.most_common_freqs, e.most_common_base_freqs
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false
+\gx
+-[ RECORD 1 ]----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+n_distinct | {"2, 3": 4, "2, -1": 4, "2, -2": 4, "3, -1": 4, "3, -2": 4, "-1, -2": 3, "2, 3, -1": 4, "2, 3, -2": 4, "2, -1, -2": 4, "3, -1, -2": 4, "2, 3, -1, -2": 4}
+dependencies | {"2 => 3": 1.000000, "2 => -1": 1.000000, "2 => -2": 1.000000, "3 => 2": 1.000000, "3 => -1": 1.000000, "3 => -2": 1.000000, "-1 => 2": 0.500000, "-1 => 3": 0.500000, "-1 => -2": 1.000000, "-2 => 2": 0.500000, "-2 => 3": 0.500000, "-2 => -1": 1.000000, "2, 3 => -1": 1.000000, "2, 3 => -2": 1.000000, "2, -1 => 3": 1.000000, "2, -1 => -2": 1.000000, "2, -2 => 3": 1.000000, "2, -2 => -1": 1.000000, "3, -1 => 2": 1.000000, "3, -1 => -2": 1.000000, "3, -2 => 2": 1.000000, "3, -2 => -1": 1.000000, "-1, -2 => 2": 0.500000, "-1, -2 => 3": 0.500000, "2, 3, -1 => -2": 1.000000, "2, 3, -2 => -1": 1.000000, "2, -1, -2 => 3": 1.000000, "3, -1, -2 => 2": 1.000000}
+most_common_vals |
+most_common_val_nulls |
+most_common_freqs |
+most_common_base_freqs |
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_vals => '{{four,NULL,0,NULL},{one,"(1,1.1,ONE,01-01-2001,\"{\"\"xkey\"\": \"\"xval\"\"}\")",1,2},{tre,"(3,3.3,TRE,03-03-2003,)",-1,3},{two,"(2,2.2,TWO,02-02-2002,\"[true, 4, \"\"six\"\"]\")",1,2}}'
+ );
+ERROR: MCV parameters "most_common_vals", "most_common_val_nulls", "most_common_freqs", and "most_common_base_freqs" must be all specified if any are specified
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_val_nulls => '{{f,t,f,t},{f,f,f,f},{f,f,f,f},{f,f,f,f}}'
+ );
+ERROR: MCV parameters "most_common_vals", "most_common_val_nulls", "most_common_freqs", and "most_common_base_freqs" must be all specified if any are specified
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_freqs => '{0.25,0.25,0.25,0.25}'
+ );
+ERROR: MCV parameters "most_common_vals", "most_common_val_nulls", "most_common_freqs", and "most_common_base_freqs" must be all specified if any are specified
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_base_freqs => '{0.00390625,0.015625,0.00390625,0.015625}'
+ );
+ERROR: MCV parameters "most_common_vals", "most_common_val_nulls", "most_common_freqs", and "most_common_base_freqs" must be all specified if any are specified
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_vals => '{{four,NULL,0,NULL},{one,"(1,1.1,ONE,01-01-2001,\"{\"\"xkey\"\": \"\"xval\"\"}\")",1,2},{tre,"(3,3.3,TRE,03-03-2003,)",-1,3},{two,"(2,2.2,TWO,02-02-2002,\"[true, 4, \"\"six\"\"]\")",1,2}}',
+ most_common_val_nulls => '{{f,t,f,t},{f,f,f,f},{f,f,f,f},{f,f,f,f}}',
+ most_common_freqs => '{0.25,0.25,0.25,0.25}',
+ most_common_base_freqs => '{0.00390625,0.015625,0.00390625,0.015625}'
+ );
+ pg_set_extended_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ e.n_distinct, e.dependencies, e.most_common_vals, e.most_common_val_nulls,
+ e.most_common_freqs, e.most_common_base_freqs
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false
+\gx
+-[ RECORD 1 ]----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+n_distinct | {"2, 3": 4, "2, -1": 4, "2, -2": 4, "3, -1": 4, "3, -2": 4, "-1, -2": 3, "2, 3, -1": 4, "2, 3, -2": 4, "2, -1, -2": 4, "3, -1, -2": 4, "2, 3, -1, -2": 4}
+dependencies | {"2 => 3": 1.000000, "2 => -1": 1.000000, "2 => -2": 1.000000, "3 => 2": 1.000000, "3 => -1": 1.000000, "3 => -2": 1.000000, "-1 => 2": 0.500000, "-1 => 3": 0.500000, "-1 => -2": 1.000000, "-2 => 2": 0.500000, "-2 => 3": 0.500000, "-2 => -1": 1.000000, "2, 3 => -1": 1.000000, "2, 3 => -2": 1.000000, "2, -1 => 3": 1.000000, "2, -1 => -2": 1.000000, "2, -2 => 3": 1.000000, "2, -2 => -1": 1.000000, "3, -1 => 2": 1.000000, "3, -1 => -2": 1.000000, "3, -2 => 2": 1.000000, "3, -2 => -1": 1.000000, "-1, -2 => 2": 0.500000, "-1, -2 => 3": 0.500000, "2, 3, -1 => -2": 1.000000, "2, 3, -2 => -1": 1.000000, "2, -1, -2 => 3": 1.000000, "3, -1, -2 => 2": 1.000000}
+most_common_vals | {{four,NULL,0,NULL},{one,"(1,1.1,ONE,01-01-2001,\"{\"\"xkey\"\": \"\"xval\"\"}\")",1,2},{tre,"(3,3.3,TRE,03-03-2003,)",-1,3},{two,"(2,2.2,TWO,02-02-2002,\"[true, 4, \"\"six\"\"]\")",1,2}}
+most_common_val_nulls | {{f,t,f,t},{f,f,f,f},{f,f,f,f},{f,f,f,f}}
+most_common_freqs | {0.25,0.25,0.25,0.25}
+most_common_base_freqs | {0.00390625,0.015625,0.00390625,0.015625}
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ exprs => '{{0,4,-0.75,"{1}","{0.5}","{-1,0}",-0.6,NULL,NULL,NULL},{0.25,4,-0.5,"{2}","{0.5}",NULL,1,NULL,NULL,NULL}}'
+ );
+ pg_set_extended_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ e.inherited, e.null_frac, e.avg_width, e.n_distinct, e.most_common_vals,
+ e.most_common_freqs, e.histogram_bounds, e.correlation,
+ e.most_common_elems, e.most_common_elem_freqs, e.elem_count_histogram
+FROM pg_stats_ext_exprs AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+and e.inherited = false
+\gx
+-[ RECORD 1 ]----------+-------
+inherited | f
+null_frac | 0
+avg_width | 4
+n_distinct | -0.75
+most_common_vals | {1}
+most_common_freqs | {0.5}
+histogram_bounds | {-1,0}
+correlation | -0.6
+most_common_elems |
+most_common_elem_freqs |
+elem_count_histogram |
+-[ RECORD 2 ]----------+-------
+inherited | f
+null_frac | 0.25
+avg_width | 4
+n_distinct | -0.5
+most_common_vals | {2}
+most_common_freqs | {0.5}
+histogram_bounds |
+correlation | 1
+most_common_elems |
+most_common_elem_freqs |
+elem_count_histogram |
+
+SELECT
+ pg_catalog.pg_clear_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false);
+ pg_clear_extended_stats
+-------------------------
+
+(1 row)
+
+SELECT COUNT(*)
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false;
+ count
+-------
+ 0
+(1 row)
+
+SELECT COUNT(*)
+FROM pg_stats_ext_exprs AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false;
+ count
+-------
+ 0
+(1 row)
+
+--
+-- Copy stats from test_stat to test_stat_clone
+--
+SELECT
+ e.statistics_name,
+ pg_catalog.pg_restore_extended_stats(
+ 'statistics_schemaname', e.statistics_schemaname::regnamespace,
+ 'statistics_name', 'test_stat_clone'::name,
+ 'inherited', e.inherited,
+ 'n_distinct', e.n_distinct,
+ 'dependencies', e.dependencies,
+ 'most_common_vals', e.most_common_vals,
+ 'most_common_val_nulls', e.most_common_val_nulls,
+ 'most_common_freqs', e.most_common_freqs,
+ 'most_common_base_freqs', e.most_common_base_freqs,
+ 'exprs', x.exprs
+ )
+FROM pg_stats_ext AS e
+CROSS JOIN LATERAL (
+ SELECT
+ array_agg(
+ ARRAY[ee.null_frac::text, ee.avg_width::text,
+ ee.n_distinct::text, ee.most_common_vals::text,
+ ee.most_common_freqs::text, ee.histogram_bounds::text,
+ ee.correlation::text, ee.most_common_elems::text,
+ ee.most_common_elem_freqs::text,
+ ee.elem_count_histogram::text])
+ FROM pg_stats_ext_exprs AS ee
+ WHERE ee.statistics_schemaname = e.statistics_schemaname
+ AND ee.statistics_name = e.statistics_name
+ AND ee.inherited = e.inherited
+ ) AS x(exprs)
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat';
+ statistics_name | pg_restore_extended_stats
+-----------------+---------------------------
+ test_stat | t
+(1 row)
+
+SELECT o.inherited,
+ o.n_distinct, o.dependencies, o.most_common_vals,
+ o.most_common_val_nulls, o.most_common_freqs,
+ o.most_common_base_freqs
+FROM pg_stats_ext AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat'
+EXCEPT
+SELECT n.inherited,
+ n.n_distinct, n.dependencies, n.most_common_vals,
+ n.most_common_val_nulls, n.most_common_freqs,
+ n.most_common_base_freqs
+FROM pg_stats_ext AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone';
+ inherited | n_distinct | dependencies | most_common_vals | most_common_val_nulls | most_common_freqs | most_common_base_freqs
+-----------+------------+--------------+------------------+-----------------------+-------------------+------------------------
+(0 rows)
+
+SELECT n.inherited,
+ n.n_distinct, n.dependencies, n.most_common_vals,
+ n.most_common_val_nulls, n.most_common_freqs,
+ n.most_common_base_freqs
+FROM pg_stats_ext AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone'
+EXCEPT
+SELECT o.inherited,
+ o.n_distinct, o.dependencies, o.most_common_vals,
+ o.most_common_val_nulls, o.most_common_freqs,
+ o.most_common_base_freqs
+FROM pg_stats_ext AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat';
+ inherited | n_distinct | dependencies | most_common_vals | most_common_val_nulls | most_common_freqs | most_common_base_freqs
+-----------+------------+--------------+------------------+-----------------------+-------------------+------------------------
+(0 rows)
+
+SELECT o.inherited,
+ o.null_frac, o.avg_width, o.n_distinct,
+ o.most_common_vals::text AS most_common_vals,
+ o.most_common_freqs,
+ o.histogram_bounds::text AS histogram_bounds,
+ o.correlation,
+ o.most_common_elems::text AS most_common_elems,
+ o.most_common_elem_freqs, o.elem_count_histogram
+FROM pg_stats_ext_exprs AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat'
+EXCEPT
+SELECT n.inherited,
+ n.null_frac, n.avg_width, n.n_distinct,
+ n.most_common_vals::text AS most_common_vals,
+ n.most_common_freqs,
+ n.histogram_bounds::text AS histogram_bounds,
+ n.correlation,
+ n.most_common_elems::text AS most_common_elems,
+ n.most_common_elem_freqs, n.elem_count_histogram
+FROM pg_stats_ext_exprs AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone';
+ inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram
+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------
+(0 rows)
+
+SELECT n.inherited,
+ n.null_frac, n.avg_width, n.n_distinct,
+ n.most_common_vals::text AS most_common_vals,
+ n.most_common_freqs,
+ n.histogram_bounds::text AS histogram_bounds,
+ n.correlation,
+ n.most_common_elems::text AS most_common_elems,
+ n.most_common_elem_freqs, n.elem_count_histogram
+FROM pg_stats_ext_exprs AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone'
+EXCEPT
+SELECT o.inherited,
+ o.null_frac, o.avg_width, o.n_distinct,
+ o.most_common_vals::text AS most_common_vals,
+ o.most_common_freqs,
+ o.histogram_bounds::text AS histogram_bounds,
+ o.correlation,
+ o.most_common_elems::text AS most_common_elems,
+ o.most_common_elem_freqs, o.elem_count_histogram
+FROM pg_stats_ext_exprs AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat';
+ inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram
+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------
+(0 rows)
+
DROP SCHEMA stats_import CASCADE;
NOTICE: drop cascades to 6 other objects
DETAIL: drop cascades to type stats_import.complex_type
diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql
index d3058bf8f6..98aa934d12 100644
--- a/src/test/regress/sql/stats_import.sql
+++ b/src/test/regress/sql/stats_import.sql
@@ -1062,6 +1062,9 @@ SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+CREATE STATISTICS stats_import.test_stat ON name, comp, lower(arange), array_length(tags,1)
+FROM stats_import.test;
+
-- Generate statistics on table with data
ANALYZE stats_import.test;
@@ -1070,6 +1073,9 @@ CREATE TABLE stats_import.test_clone ( LIKE stats_import.test )
CREATE INDEX is_odd_clone ON stats_import.test_clone(((comp).a % 2 = 1));
+CREATE STATISTICS stats_import.test_stat_clone ON name, comp, lower(arange), array_length(tags,1)
+FROM stats_import.test_clone;
+
--
-- Copy stats from test to test_clone, and is_odd to is_odd_clone
--
@@ -1381,4 +1387,242 @@ FROM pg_statistic s
JOIN pg_attribute a ON a.attrelid = s.starelid AND a.attnum = s.staattnum
WHERE s.starelid = 'stats_import.is_odd'::regclass;
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ n_distinct => '{"2, 3": 4, "2, -1": 4, "2, -2": 4, "3, -1": 4, "3, -2": 4, "-1, -2": 3, "2, 3, -1": 4, "2, 3, -2": 4, "2, -1, -2": 4, "3, -1, -2": 4, "2, 3, -1, -2": 4}'::pg_ndistinct
+ );
+
+SELECT
+ e.n_distinct, e.dependencies, e.most_common_vals, e.most_common_val_nulls,
+ e.most_common_freqs, e.most_common_base_freqs
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false
+\gx
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ dependencies => '{"2 => 3": 1.000000, "2 => -1": 1.000000, "2 => -2": 1.000000, "3 => 2": 1.000000, "3 => -1": 1.000000, "3 => -2": 1.000000, "-1 => 2": 0.500000, "-1 => 3": 0.500000, "-1 => -2": 1.000000, "-2 => 2": 0.500000, "-2 => 3": 0.500000, "-2 => -1": 1.000000, "2, 3 => -1": 1.000000, "2, 3 => -2": 1.000000, "2, -1 => 3": 1.000000, "2, -1 => -2": 1.000000, "2, -2 => 3": 1.000000, "2, -2 => -1": 1.000000, "3, -1 => 2": 1.000000, "3, -1 => -2": 1.000000, "3, -2 => 2": 1.000000, "3, -2 => -1": 1.000000, "-1, -2 => 2": 0.500000, "-1, -2 => 3": 0.500000, "2, 3, -1 => -2": 1.000000, "2, 3, -2 => -1": 1.000000, "2, -1, -2 => 3": 1.000000, "3, -1, -2 => 2": 1.000000}'::pg_dependencies
+ );
+
+SELECT
+ e.n_distinct, e.dependencies, e.most_common_vals, e.most_common_val_nulls,
+ e.most_common_freqs, e.most_common_base_freqs
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false
+\gx
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_vals => '{{four,NULL,0,NULL},{one,"(1,1.1,ONE,01-01-2001,\"{\"\"xkey\"\": \"\"xval\"\"}\")",1,2},{tre,"(3,3.3,TRE,03-03-2003,)",-1,3},{two,"(2,2.2,TWO,02-02-2002,\"[true, 4, \"\"six\"\"]\")",1,2}}'
+ );
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_val_nulls => '{{f,t,f,t},{f,f,f,f},{f,f,f,f},{f,f,f,f}}'
+ );
+
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_freqs => '{0.25,0.25,0.25,0.25}'
+ );
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_base_freqs => '{0.00390625,0.015625,0.00390625,0.015625}'
+ );
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ most_common_vals => '{{four,NULL,0,NULL},{one,"(1,1.1,ONE,01-01-2001,\"{\"\"xkey\"\": \"\"xval\"\"}\")",1,2},{tre,"(3,3.3,TRE,03-03-2003,)",-1,3},{two,"(2,2.2,TWO,02-02-2002,\"[true, 4, \"\"six\"\"]\")",1,2}}',
+ most_common_val_nulls => '{{f,t,f,t},{f,f,f,f},{f,f,f,f},{f,f,f,f}}',
+ most_common_freqs => '{0.25,0.25,0.25,0.25}',
+ most_common_base_freqs => '{0.00390625,0.015625,0.00390625,0.015625}'
+ );
+
+SELECT
+ e.n_distinct, e.dependencies, e.most_common_vals, e.most_common_val_nulls,
+ e.most_common_freqs, e.most_common_base_freqs
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false
+\gx
+
+SELECT
+ pg_catalog.pg_set_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false,
+ exprs => '{{0,4,-0.75,"{1}","{0.5}","{-1,0}",-0.6,NULL,NULL,NULL},{0.25,4,-0.5,"{2}","{0.5}",NULL,1,NULL,NULL,NULL}}'
+ );
+
+SELECT
+ e.inherited, e.null_frac, e.avg_width, e.n_distinct, e.most_common_vals,
+ e.most_common_freqs, e.histogram_bounds, e.correlation,
+ e.most_common_elems, e.most_common_elem_freqs, e.elem_count_histogram
+FROM pg_stats_ext_exprs AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+and e.inherited = false
+\gx
+
+SELECT
+ pg_catalog.pg_clear_extended_stats(
+ statistics_schemaname => 'stats_import'::regnamespace,
+ statistics_name => 'test_stat_clone'::name,
+ inherited => false);
+
+SELECT COUNT(*)
+FROM pg_stats_ext AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false;
+
+SELECT COUNT(*)
+FROM pg_stats_ext_exprs AS e
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat_clone'
+AND e.inherited = false;
+
+--
+-- Copy stats from test_stat to test_stat_clone
+--
+SELECT
+ e.statistics_name,
+ pg_catalog.pg_restore_extended_stats(
+ 'statistics_schemaname', e.statistics_schemaname::regnamespace,
+ 'statistics_name', 'test_stat_clone'::name,
+ 'inherited', e.inherited,
+ 'n_distinct', e.n_distinct,
+ 'dependencies', e.dependencies,
+ 'most_common_vals', e.most_common_vals,
+ 'most_common_val_nulls', e.most_common_val_nulls,
+ 'most_common_freqs', e.most_common_freqs,
+ 'most_common_base_freqs', e.most_common_base_freqs,
+ 'exprs', x.exprs
+ )
+FROM pg_stats_ext AS e
+CROSS JOIN LATERAL (
+ SELECT
+ array_agg(
+ ARRAY[ee.null_frac::text, ee.avg_width::text,
+ ee.n_distinct::text, ee.most_common_vals::text,
+ ee.most_common_freqs::text, ee.histogram_bounds::text,
+ ee.correlation::text, ee.most_common_elems::text,
+ ee.most_common_elem_freqs::text,
+ ee.elem_count_histogram::text])
+ FROM pg_stats_ext_exprs AS ee
+ WHERE ee.statistics_schemaname = e.statistics_schemaname
+ AND ee.statistics_name = e.statistics_name
+ AND ee.inherited = e.inherited
+ ) AS x(exprs)
+WHERE e.statistics_schemaname = 'stats_import'
+AND e.statistics_name = 'test_stat';
+
+SELECT o.inherited,
+ o.n_distinct, o.dependencies, o.most_common_vals,
+ o.most_common_val_nulls, o.most_common_freqs,
+ o.most_common_base_freqs
+FROM pg_stats_ext AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat'
+EXCEPT
+SELECT n.inherited,
+ n.n_distinct, n.dependencies, n.most_common_vals,
+ n.most_common_val_nulls, n.most_common_freqs,
+ n.most_common_base_freqs
+FROM pg_stats_ext AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone';
+
+SELECT n.inherited,
+ n.n_distinct, n.dependencies, n.most_common_vals,
+ n.most_common_val_nulls, n.most_common_freqs,
+ n.most_common_base_freqs
+FROM pg_stats_ext AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone'
+EXCEPT
+SELECT o.inherited,
+ o.n_distinct, o.dependencies, o.most_common_vals,
+ o.most_common_val_nulls, o.most_common_freqs,
+ o.most_common_base_freqs
+FROM pg_stats_ext AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat';
+
+SELECT o.inherited,
+ o.null_frac, o.avg_width, o.n_distinct,
+ o.most_common_vals::text AS most_common_vals,
+ o.most_common_freqs,
+ o.histogram_bounds::text AS histogram_bounds,
+ o.correlation,
+ o.most_common_elems::text AS most_common_elems,
+ o.most_common_elem_freqs, o.elem_count_histogram
+FROM pg_stats_ext_exprs AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat'
+EXCEPT
+SELECT n.inherited,
+ n.null_frac, n.avg_width, n.n_distinct,
+ n.most_common_vals::text AS most_common_vals,
+ n.most_common_freqs,
+ n.histogram_bounds::text AS histogram_bounds,
+ n.correlation,
+ n.most_common_elems::text AS most_common_elems,
+ n.most_common_elem_freqs, n.elem_count_histogram
+FROM pg_stats_ext_exprs AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone';
+
+SELECT n.inherited,
+ n.null_frac, n.avg_width, n.n_distinct,
+ n.most_common_vals::text AS most_common_vals,
+ n.most_common_freqs,
+ n.histogram_bounds::text AS histogram_bounds,
+ n.correlation,
+ n.most_common_elems::text AS most_common_elems,
+ n.most_common_elem_freqs, n.elem_count_histogram
+FROM pg_stats_ext_exprs AS n
+WHERE n.statistics_schemaname = 'stats_import'
+AND n.statistics_name = 'test_stat_clone'
+EXCEPT
+SELECT o.inherited,
+ o.null_frac, o.avg_width, o.n_distinct,
+ o.most_common_vals::text AS most_common_vals,
+ o.most_common_freqs,
+ o.histogram_bounds::text AS histogram_bounds,
+ o.correlation,
+ o.most_common_elems::text AS most_common_elems,
+ o.most_common_elem_freqs, o.elem_count_histogram
+FROM pg_stats_ext_exprs AS o
+WHERE o.statistics_schemaname = 'stats_import'
+AND o.statistics_name = 'test_stat';
+
DROP SCHEMA stats_import CASCADE;
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 47370e581a..d74f23eda2 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -30344,6 +30344,130 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset
'inherited', false,
'avg_width', 125::integer,
'null_frac', 0.5::real);
+</programlisting>
+ </para>
+ <para>
+ Minor errors are reported as a <literal>WARNING</literal> and
+ ignored, and remaining statistics will still be restored. If all
+ specified statistics are successfully restored, return
+ <literal>true</literal>, otherwise <literal>false</literal>.
+ </para>
+ </entry>
+ </row>
+ <row>
+ <entry role="func_table_entry">
+ <para role="func_signature">
+ <indexterm>
+ <primary>pg_set_extended_stats</primary>
+ </indexterm>
+ <function>pg_set_extended_stats</function> (
+ <parameter>statistics_schemaname</parameter> <type>regnamespace</type>,
+ <parameter>statistics_name</parameter> <type>name</type>,
+ <parameter>inherited</parameter> <type>boolean</type>,
+ <optional>, <parameter>n_distinct</parameter> <type>pg_ndistinct</type></optional>,
+ <optional>, <parameter>dependencies</parameter> <type>pg_dependencies</type></optional>,
+ <optional>, <parameter>most_common_vals</parameter> <type>text[]</type></optional>,
+ <optional>, <parameter>most_common_val_nulls</parameter> <type>boolean[]</type></optional>,
+ <optional>, <parameter>most_common_freqs</parameter> <type>double precision[]</type> </optional>,
+ <optional>, <parameter>most_common_base_freqs</parameter> <type>double precision[]</type> </optional>,
+ <optional>, <parameter>exprs</parameter> <type>text[]</type> </optional> )
+ <returnvalue>void</returnvalue>
+ </para>
+ <para>
+ Creates or updates statistics for the given statistics object to the
+ specified values. The parameters correspond to attributes of the same
+ name found in the view <link
+ linkend="view-pg-stats-ext"><structname>pg_stats_ext</structname></link>
+ and <link linkend="view-pg-stats-ext-exprs"><structname>pg_stats_ext_exprs</structname></link>,
+ except for <parameter>exprs</parameter> which corresponds to
+ <structfield>stxexpr</structfield> on
+ <link linkend="catalog-pg-statistic-ext-data"><structname>pg_statistic_ext_data</structname></link>
+ as a two-dimensional text array. The outer dimension represents a single
+ <structname>pg_statistic</structname> object, and the inner dimension elements are the
+ statistics fields from <link linkend="view-pg-stats"><structname>pg_stats</structname></link>
+ (currently <structfield>null_frac</structfield> through
+ <structfield>elem_count_histogram</structfield>).
+ </para>
+ <para>
+ Optional parameters default to <literal>NULL</literal>, which leave
+ the corresponding statistic unchanged.
+ </para>
+ <para>
+ Ordinarily, these statistics are collected automatically or updated
+ as a part of <xref linkend="sql-vacuum"/> or <xref
+ linkend="sql-analyze"/>, so it's not necessary to call this
+ function. However, it may be useful when testing the effects of
+ statistics on the planner to understand or anticipate plan changes.
+ </para>
+ <para>
+ The caller must have the <literal>MAINTAIN</literal> privilege on
+ the table or be the owner of the database.
+ </para>
+ </entry>
+ </row>
+
+ <row>
+ <entry role="func_table_entry">
+ <para role="func_signature">
+ <indexterm>
+ <primary>pg_clear_extended_stats</primary>
+ </indexterm>
+ <function>pg_clear_extended_stats</function> (
+ <parameter>statistics_schemaname</parameter> <type>regnamespace</type>,
+ <parameter>statistics_name</parameter> <type>name</type>,
+ <parameter>inherited</parameter> <type>boolean</type> )
+ <returnvalue>void</returnvalue>
+ </para>
+ <para>
+ Clears statistics for the given statistics object, as
+ though the object was newly created.
+ </para>
+ <para>
+ The caller must have the <literal>MAINTAIN</literal> privilege on
+ the table or be the owner of the database.
+ </para>
+ </entry>
+ </row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_restore_extended_stats</primary>
+ </indexterm>
+ <function>pg_restore_extended_stats</function> (
+ <literal>VARIADIC</literal> <parameter>kwargs</parameter> <type>"any"</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Similar to <function>pg_set_extended_stats()</function>, but
+ intended for bulk restore of extended statistics. The tracked
+ statistics may change from version to version, so the primary purpose
+ of this function is to maintain a consistent function signature to
+ avoid errors when restoring statistics from previous versions.
+ </para>
+ <para>
+ Arguments are passed as pairs of <replaceable>argname</replaceable>
+ and <replaceable>argvalue</replaceable>, where
+ <replaceable>argname</replaceable> corresponds to a named argument in
+ <function>pg_set_extended_stats()</function> and
+ <replaceable>argvalue</replaceable> is of the corresponding type.
+ </para>
+ <para>
+ Additionally, this function supports argument name
+ <literal>version</literal> of type <type>integer</type>, which
+ specifies the version from which the statistics originated, improving
+ interpretation of older statistics.
+ </para>
+ <para>
+ For example, to set the <structname>n_distinct</structname> and
+ <structname>exprs</structname> for the object
+ <structname>my_extended_stat</structname>:
+<programlisting>
+ SELECT pg_restore_extended_stats(
+ 'statistics_schemaname', 'public'::regnamespace,
+ 'statistics_name', 'my_extended_stat'::name,
+ 'n_distinct', '{"2, 3": 4, "2, -1": 4, "2, -2": 4, "3, -1": 4, "3, -2": 4, "-1, -2": 3, "2, 3, -1": 4, "2, 3, -2": 4, "2, -1, -2": 4, "3, -1, -2": 4, "2, 3, -1, -2": 4}'::pg_ndistinct,
+ 'exprs' => '{{0,4,-0.75,"{1}","{0.5}","{-1,0}",-0.6,NULL,NULL,NULL},{0.25,4,-0.5,"{2}","{0.5}",NULL,1,NULL,NULL,NULL}}');
</programlisting>
</para>
<para>
--
2.47.1
[text/x-patch] v37-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch (51.9K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/7-v37-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From 06047840b9125c6e2b58c0c4ec4531c291ea59d8 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v37 01/11] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute. These
statements will restore the statistics of the current system onto the
destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, staistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index and Materialized View
statistics are dumped in the post-data section.
Add --no-data option.
This option is useful for situations where someone wishes to test
query plans from a production database without copying production data.
This also makes the corresponding change to the simulated pg_upgrade in
the TAP tests for pg_dump.
This checks that dumping statistics is now the default, and that
--no-statistics will suppress statistics.
Add --no-schema option to pg_dump, etc.
Previously, users could use --data-only when they wanted to suppress
schema from a dump. However, that no longer makes sense now that the
data/schema binary has become the data/schema/statistics trinary.
---
src/bin/pg_dump/pg_backup.h | 10 +-
src/bin/pg_dump/pg_backup_archiver.c | 8 +
src/bin/pg_dump/pg_dump.c | 385 ++++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 9 +
src/bin/pg_dump/pg_dump_sort.c | 32 ++-
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 27 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
src/bin/pg_dump/t/002_pg_dump.pl | 28 +-
src/bin/pg_upgrade/dump.c | 6 +-
src/bin/pg_upgrade/option.c | 12 +
src/bin/pg_upgrade/pg_upgrade.h | 1 +
doc/src/sgml/ref/pg_dump.sgml | 69 +++--
doc/src/sgml/ref/pg_dumpall.sgml | 38 +++
doc/src/sgml/ref/pg_restore.sgml | 51 +++-
doc/src/sgml/ref/pgupgrade.sgml | 18 ++
src/tools/pgindent/typedefs.list | 1 +
17 files changed, 688 insertions(+), 30 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b2..3fa1474fad 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,9 +110,12 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +163,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -179,8 +183,11 @@ typedef struct _dumpOptions
int column_inserts;
int if_exists;
int no_comments;
- int no_security_labels;
+ int no_data;
int no_publications;
+ int no_schema;
+ int no_security_labels;
+ int no_statistics;
int no_subscriptions;
int no_toast_compression;
int no_unlogged_table_data;
@@ -208,6 +215,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844..db1d20d282 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -186,6 +186,9 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
@@ -2962,6 +2965,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's a stats dump, maybe ignore it */
+ if (ropt->no_statistics && strcmp(te->desc, "STATISTICS DATA") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +2998,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS DATA") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 8f73a5df95..0f4976bc77 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -430,6 +430,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -466,6 +467,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -492,8 +494,12 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -539,7 +545,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:PRsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -613,6 +619,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -784,6 +794,17 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -798,8 +819,9 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
@@ -1099,6 +1121,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1177,7 +1200,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1190,11 +1213,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1219,8 +1243,11 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
+ printf(_(" --no-schema do not dump schema\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6777,6 +6804,43 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+ info->postponed_def = false;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7154,6 +7218,7 @@ getTables(Archive *fout, int *numTables)
/* Tables have data */
tblinfo[i].dobj.components |= DUMP_COMPONENT_DATA;
+ tblinfo[i].dobj.components |= DUMP_COMPONENT_STATISTICS;
/* Mark whether table has an ACL */
if (!PQgetisnull(res, i, i_relacl))
@@ -7202,6 +7267,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7648,11 +7715,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7675,7 +7745,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7709,6 +7786,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10296,6 +10375,287 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * Decide which section to use based on the relkind of the parent object.
+ *
+ * NB: materialized views may be postponed from SECTION_PRE_DATA to
+ * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the
+ * matview stats will also be postponed to SECTION_POST_DATA. See
+ * repairMatViewBoundaryMultiLoop().
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+ switch (rsinfo->relkind)
+ {
+ case RELKIND_RELATION:
+ case RELKIND_PARTITIONED_TABLE:
+ case RELKIND_MATVIEW:
+ return SECTION_DATA;
+ case RELKIND_INDEX:
+ case RELKIND_PARTITIONED_INDEX:
+ return SECTION_POST_DATA;
+ default:
+ pg_fatal("cannot dump statistics for relation kind '%c'",
+ rsinfo->relkind);
+ }
+
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = rsinfo->postponed_def ?
+ SECTION_POST_DATA : statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = dobj->dependencies,
+ .nDeps = dobj->nDeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10744,6 +11104,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -17183,6 +17546,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo)
free(indstatvalsarray);
}
+ /* Comments and stats share same .dep */
+
/* Dump Index Comments */
if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT)
dumpComment(fout, "INDEX", qindxname,
@@ -18970,6 +19335,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index f62b564ed1..4edd88a54b 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -84,6 +84,7 @@ typedef enum
DO_PUBLICATION_TABLE_IN_SCHEMA,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
+ DO_REL_STATS,
} DumpableObjectType;
/*
@@ -109,6 +110,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -429,6 +431,13 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'v', 'c', etc */
+ bool postponed_def;
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index dc9a28924b..3a3602e3d2 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -801,11 +801,21 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj,
{
/* remove boundary's dependency on object after it in loop */
removeObjectDependency(boundaryobj, nextobj->dumpId);
- /* if that object is a matview, mark it as postponed into post-data */
+ /*
+ * If that object is a matview or matview status, mark it as postponed into
+ * post-data.
+ */
if (nextobj->objType == DO_TABLE)
{
TableInfo *nextinfo = (TableInfo *) nextobj;
+ if (nextinfo->relkind == RELKIND_MATVIEW)
+ nextinfo->postponed_def = true;
+ }
+ else if (nextobj->objType == DO_REL_STATS)
+ {
+ RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj;
+
if (nextinfo->relkind == RELKIND_MATVIEW)
nextinfo->postponed_def = true;
}
@@ -1018,6 +1028,21 @@ repairDependencyLoop(DumpableObject **loop,
{
DumpableObject *nextobj;
+ nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
+ repairMatViewBoundaryMultiLoop(loop[j], nextobj);
+ return;
+ }
+ }
+ }
+ else if (loop[i]->objType == DO_REL_STATS &&
+ ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW)
+ {
+ for (j = 0; j < nLoop; j++)
+ {
+ if (loop[j]->objType == DO_POST_DATA_BOUNDARY)
+ {
+ DumpableObject *nextobj;
+
nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
repairMatViewBoundaryMultiLoop(loop[j], nextobj);
return;
@@ -1500,6 +1525,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 396f79781c..7effb70490 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 88ae39d938..aa6db4fcc9 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,7 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -71,8 +72,11 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_comments = 0;
+ static int no_data = 0;
static int no_publications = 0;
+ static int no_schema = 0;
static int no_security_labels = 0;
+ static int no_statistics = 0;
static int no_subscriptions = 0;
static int strict_names = 0;
bool data_only = false;
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'P'},
/*
* the following options don't have an equivalent short option letter
@@ -124,9 +129,12 @@ main(int argc, char **argv)
{"transaction-size", required_argument, NULL, 5},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
@@ -271,6 +279,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +355,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +378,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpSchema = (!no_schema && !data_only && !statistics_only);
+ opts->dumpData = (!no_data && !schema_only && !statistics_only);
+ opts->dumpStatistics = (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +392,8 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
@@ -484,6 +503,7 @@ usage(const char *progname)
printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n"));
printf(_(" -T, --trigger=NAME restore named trigger\n"));
printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only restore only the statistics, not schema or data\n"));
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
@@ -491,10 +511,13 @@ usage(const char *progname)
" in FILENAME\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comment commands\n"));
+ printf(_(" --no-data do not restore data\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 214240f1ae..f29da06ed2 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index bf65d44b94..91f34e2dc0 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -66,7 +66,7 @@ my %pgdump_runs = (
'--format=custom',
"--file=$tempdir/binary_upgrade.dump",
'-w',
- '--schema-only',
+ '--no-data',
'--binary-upgrade',
'-d', 'postgres', # alternative way to specify database
],
@@ -645,6 +645,12 @@ my %pgdump_runs = (
'--schema=dump_test', '-b', '-B', '--no-sync', 'postgres',
],
+ },
+ no_statistics => {
+ dump_cmd => [
+ 'pg_dump', "--file=$tempdir/no_statistics.sql",
+ '--no-sync', '--no-statistics', 'postgres',
+ ],
},);
###############################################################
@@ -711,6 +717,7 @@ my %full_runs = (
no_large_objects => 1,
no_owner => 1,
no_privs => 1,
+ no_statistics => 1,
no_table_access_method => 1,
pg_dumpall_dbprivs => 1,
pg_dumpall_exclude => 1,
@@ -4586,6 +4593,25 @@ my %tests = (
},
},
+ # 'statistics_import' => {
+ # create_sql => '
+ # CREATE MATERIALIZED VIEW dump_test.has_stats
+ # AS SELECT g.g AS x, g.g / 2 AS y FROM generate_series(1,100) AS g(g);
+ # ANALYZE dump_test.has_stats;',
+ # regexp => qr/pg_catalog.pg_restore_attribute_stats/,
+ # like => {
+ # %full_runs,
+ # test_schema_plus_large_objects => 1,
+ # },
+ # unlike => {
+ # exclude_dump_test_schema => 1,
+ # no_statistics => 1,
+ # only_dump_test_schema => 1,
+ # schema_only => 1,
+ # section_post_data => 1,
+ # },
+ #},
+
# CREATE TABLE with partitioned table and various AMs. One
# partition uses the same default as the parent, and a second
# uses its own AM.
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8ce0fa3020..a29cd2cca9 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 108eb7a1ba..3b6c7ec994 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -306,7 +316,9 @@ usage(void)
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
+ printf(_(" --no-statistics do not import statistics from old cluster\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statistics import statistics from old cluster (default)\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 0cdd675e4f..3fe111fbde 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d66e901f51..5e58f24d21 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,13 +141,11 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
- switch is therefore only useful to add large objects to dumps
- where a specific schema or table has been requested. Note that
- large objects are considered data and therefore will be included when
- <option>--data-only</option> is used, but not
- when <option>--schema-only</option> is.
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or <option>--statistics-only</option>, or
+ <option>--no-data</option> is specified. The <option>-b</option>
+ switch is therefore only useful to add large objects to dumps where a
+ specific schema or table has been requested.
</para>
</listitem>
</varlistentry>
@@ -516,10 +514,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +651,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -824,16 +835,17 @@ PostgreSQL documentation
<term><option>--exclude-table-data=<replaceable class="parameter">pattern</replaceable></option></term>
<listitem>
<para>
- Do not dump data for any tables matching <replaceable
+ Do not dump data or statistics for any tables matching <replaceable
class="parameter">pattern</replaceable>. The pattern is
interpreted according to the same rules as for <option>-t</option>.
<option>--exclude-table-data</option> can be given more than once to
- exclude tables matching any of several patterns. This option is
- useful when you need the definition of a particular table even
- though you do not need the data in it.
+ exclude tables matching any of several patterns. This option is useful
+ when you need the definition of a particular table even though you do
+ not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1080,6 +1092,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -1098,6 +1119,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index 014f279258..d423153a93 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -265,6 +265,17 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--binary-upgrade</option></term>
<listitem>
@@ -422,6 +433,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -447,6 +467,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -456,6 +485,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719..22c3c118ad 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,20 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema (data definitions) or data.
+ </para>
+ <para>
+ (Do not confuse this with the <option>--schema</option> option, which
+ uses the word <quote>schema</quote> in a different meaning.)
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -681,6 +696,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore data, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-data-for-failed-tables</option></term>
<listitem>
@@ -713,6 +738,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema (data definitions), even if
+ the archive contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -723,6 +758,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml
index 4777381dac..64a1ebd613 100644
--- a/doc/src/sgml/ref/pgupgrade.sgml
+++ b/doc/src/sgml/ref/pgupgrade.sgml
@@ -145,6 +145,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-statistics</option></term>
+ <listitem>
+ <para>
+ Restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-o</option> <replaceable class="parameter">options</replaceable></term>
<term><option>--old-options</option> <replaceable class="parameter">options</replaceable></term>
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e1c4f913f8..a1863e0263 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2394,6 +2394,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelStatsInfo
RelToCheck
RelToCluster
RelabelType
--
2.47.1
[text/x-patch] v37-0007-split-out-print_processing_notice.patch (2.4K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/8-v37-0007-split-out-print_processing_notice.patch)
download | inline diff:
From 2dbedb55639efed864d5041f004a932f9c8cf7e0 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:46:51 -0500
Subject: [PATCH v37 07/11] split out print_processing_notice
---
src/bin/scripts/vacuumdb.c | 41 +++++++++++++++++++++++---------------
1 file changed, 25 insertions(+), 16 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 9a92f65c4b..2aa55f191e 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -535,6 +535,30 @@ check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
}
+/*
+ * print the processing notice for a database.
+ */
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet)
+{
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ if (!quiet)
+ {
+ if (stage != ANALYZE_NO_STAGE)
+ printf(_("%s: processing database \"%s\": %s\n"),
+ progname, PQdb(conn), _(stage_messages[stage]));
+ else
+ printf(_("%s: vacuuming database \"%s\"\n"),
+ progname, PQdb(conn));
+ fflush(stdout);
+ }
+}
+
/*
* vacuum_one_database
*
@@ -574,11 +598,6 @@ vacuum_one_database(ConnParams *cparams,
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
"RESET default_statistics_target;"
};
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
@@ -586,17 +605,7 @@ vacuum_one_database(ConnParams *cparams,
conn = connectDatabase(cparams, progname, echo, false, true);
check_conn_options(conn, vacopts);
-
- if (!quiet)
- {
- if (stage != ANALYZE_NO_STAGE)
- printf(_("%s: processing database \"%s\": %s\n"),
- progname, PQdb(conn), _(stage_messages[stage]));
- else
- printf(_("%s: vacuuming database \"%s\"\n"),
- progname, PQdb(conn));
- fflush(stdout);
- }
+ print_processing_notice(conn, stage, progname, quiet);
/*
* Prepare the list of tables to process by querying the catalogs.
--
2.47.1
[text/x-patch] v37-0006-split-out-check_conn_options.patch (4.5K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/9-v37-0006-split-out-check_conn_options.patch)
download | inline diff:
From d14231a7cd193f37f3e99b9d9b2a582fa14b99fc Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 15:20:07 -0500
Subject: [PATCH v37 06/11] split out check_conn_options
---
src/bin/scripts/vacuumdb.c | 103 ++++++++++++++++++++-----------------
1 file changed, 57 insertions(+), 46 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 170f34d927..9a92f65c4b 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -10,6 +10,7 @@
*-------------------------------------------------------------------------
*/
+#include "libpq-fe.h"
#include "postgres_fe.h"
#include <limits.h>
@@ -459,55 +460,11 @@ escape_quotes(const char *src)
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
+ * Check connection options for compatibility with the connected database.
*/
static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+check_conn_options(PGconn *conn, vacuumingOptions *vacopts)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
- PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
- SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
- bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
- const char *stage_messages[] = {
- gettext_noop("Generating minimal optimizer statistics (1 target)"),
- gettext_noop("Generating medium optimizer statistics (10 targets)"),
- gettext_noop("Generating default (full) optimizer statistics")
- };
-
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
-
- conn = connectDatabase(cparams, progname, echo, false, true);
-
if (vacopts->disable_page_skipping && PQserverVersion(conn) < 90600)
{
PQfinish(conn);
@@ -575,6 +532,60 @@ vacuum_one_database(ConnParams *cparams,
/* skip_database_stats is used automatically if server supports it */
vacopts->skip_database_stats = (PQserverVersion(conn) >= 160000);
+}
+
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PQExpBufferData buf;
+ PQExpBufferData catalog_query;
+ PGresult *res;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ SimpleStringList dbtables = {NULL, NULL};
+ int i;
+ int ntups;
+ bool failed = false;
+ bool objects_listed = false;
+ const char *initcmd;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+ const char *stage_messages[] = {
+ gettext_noop("Generating minimal optimizer statistics (1 target)"),
+ gettext_noop("Generating medium optimizer statistics (10 targets)"),
+ gettext_noop("Generating default (full) optimizer statistics")
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
if (!quiet)
{
--
2.47.1
[text/x-patch] v37-0009-preserve-catalog-lists-across-staged-runs.patch (6.7K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/10-v37-0009-preserve-catalog-lists-across-staged-runs.patch)
download | inline diff:
From 7a8cfd54e7ed765e175ec5da130dfd8a20d97038 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 21:30:49 -0500
Subject: [PATCH v37 09/11] preserve catalog lists across staged runs
---
src/bin/scripts/vacuumdb.c | 113 ++++++++++++++++++++++++++-----------
1 file changed, 80 insertions(+), 33 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 5ae7241716..28506ca0c2 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -68,6 +68,9 @@ static void vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet);
static void vacuum_all_databases(ConnParams *cparams,
@@ -83,6 +86,14 @@ static void prepare_vacuum_command(PQExpBuffer sql, int serverVersion,
static void run_vacuum_command(PGconn *conn, const char *sql, bool echo,
const char *table);
+static void check_conn_options(PGconn *conn, vacuumingOptions *vacopts);
+
+static void
+print_processing_notice(PGconn *conn, int stage, const char *progname, bool quiet);
+
+static SimpleStringList * generate_catalog_list(PGconn *conn, vacuumingOptions *vacopts,
+ SimpleStringList *objects, bool echo, int *ntups);
+
static void help(const char *progname);
void check_objfilter(void);
@@ -386,6 +397,11 @@ main(int argc, char *argv[])
}
else
{
+ PGconn *conn;
+ int ntup;
+ SimpleStringList *found_objects;
+ int stage;
+
if (dbname == NULL)
{
if (getenv("PGDATABASE"))
@@ -397,25 +413,37 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ check_conn_options(conn, &vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
if (analyze_in_stages)
{
- int stage;
-
for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
{
- vacuum_one_database(&cparams, &vacopts,
- stage,
+ /* the last pass disconnected the conn */
+ if (stage > 0)
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
}
else
- vacuum_one_database(&cparams, &vacopts,
- ANALYZE_NO_STAGE,
+ vacuum_one_database(&cparams, &vacopts, stage,
&objects,
concurrentCons,
+ conn,
+ found_objects,
+ ntup,
progname, echo, quiet);
}
@@ -791,16 +819,17 @@ vacuum_one_database(ConnParams *cparams,
int stage,
SimpleStringList *objects,
int concurrentCons,
+ PGconn *conn,
+ SimpleStringList *dbtables,
+ int ntups,
const char *progname, bool echo, bool quiet)
{
PQExpBufferData sql;
- PGconn *conn;
SimpleStringListCell *cell;
ParallelSlotArray *sa;
- int ntups;
bool failed = false;
const char *initcmd;
- SimpleStringList *dbtables;
+
const char *stage_commands[] = {
"SET default_statistics_target=1; SET vacuum_cost_delay=0;",
"SET default_statistics_target=10; RESET vacuum_cost_delay;",
@@ -810,13 +839,6 @@ vacuum_one_database(ConnParams *cparams,
Assert(stage == ANALYZE_NO_STAGE ||
(stage >= 0 && stage < ANALYZE_NUM_STAGES));
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
-
/*
* If no rows are returned, there are no matching tables, so we are done.
*/
@@ -928,7 +950,7 @@ finish:
}
/*
- * Vacuum/analyze all connectable databases.
+ * Vacuum/analyze all ccparams->override_dbname = PQgetvalue(result, i, 0);onnectable databases.
*
* In analyze-in-stages mode, we process all databases in one stage before
* moving on to the next stage. That ensure minimal stats are available
@@ -944,8 +966,13 @@ vacuum_all_databases(ConnParams *cparams,
{
PGconn *conn;
PGresult *result;
- int stage;
int i;
+ int stage;
+
+ SimpleStringList **found_objects;
+ int *num_tuples;
+
+ stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
@@ -953,7 +980,33 @@ vacuum_all_databases(ConnParams *cparams,
echo);
PQfinish(conn);
- if (analyze_in_stages)
+ /*
+ * connect to each database, check validity of options,
+ * build the list of found objects per database,
+ * and run the first/only vacuum stage
+ */
+ found_objects = palloc(PQntuples(result) * sizeof(SimpleStringList *));
+ num_tuples = palloc(PQntuples(result) * sizeof (int));
+
+ for (i = 0; i < PQntuples(result); i++)
+ {
+ cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+ found_objects[i] = generate_catalog_list(conn, vacopts, objects, echo, &num_tuples[i]);
+
+ vacuum_one_database(cparams, vacopts,
+ stage,
+ objects,
+ concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
+ progname, echo, quiet);
+ }
+
+ if (stage != ANALYZE_NO_STAGE)
{
/*
* When analyzing all databases in stages, we analyze them all in the
@@ -963,35 +1016,29 @@ vacuum_all_databases(ConnParams *cparams,
* This means we establish several times as many connections, but
* that's a secondary consideration.
*/
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
for (i = 0; i < PQntuples(result); i++)
{
cparams->override_dbname = PQgetvalue(result, i, 0);
+ conn = connectDatabase(cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(cparams, vacopts,
stage,
objects,
concurrentCons,
+ conn,
+ found_objects[i],
+ num_tuples[i],
progname, echo, quiet);
}
}
}
- else
- {
- for (i = 0; i < PQntuples(result); i++)
- {
- cparams->override_dbname = PQgetvalue(result, i, 0);
-
- vacuum_one_database(cparams, vacopts,
- ANALYZE_NO_STAGE,
- objects,
- concurrentCons,
- progname, echo, quiet);
- }
- }
PQclear(result);
+ pfree(found_objects);
+ pfree(num_tuples);
}
/*
--
2.47.1
[text/x-patch] v37-0010-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch (1.5K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/11-v37-0010-Add-issues_sql_unlike-opposite-of-issues_sql_lik.patch)
download | inline diff:
From 3b189ae9bcaf542d37d7ca3c4e356f449d2bb5e6 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Thu, 7 Nov 2024 18:06:52 -0500
Subject: [PATCH v37 10/11] Add issues_sql_unlike, opposite of issues_sql_like
This is the same as issues_sql_like(), but the specified text is
prohibited from being in the output rather than required.
This became necessary to test that a command-line filter was in fact
filtering out certain output that a prior test required.
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 25 ++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 08b89a4cdf..7f6e5508fe 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2800,6 +2800,31 @@ sub issues_sql_like
}
=pod
+=item $node->issues_sql_unlike(cmd, prohibited_sql, test_name)
+
+Run a command on the node, then verify that $prohibited_sql does not appear
+in the server log file.
+
+=cut
+
+sub issues_sql_unlike
+{
+ local $Test::Builder::Level = $Test::Builder::Level + 1;
+
+ my ($self, $cmd, $prohibited_sql, $test_name) = @_;
+
+ local %ENV = $self->_get_env();
+
+ my $log_location = -s $self->logfile;
+
+ my $result = PostgreSQL::Test::Utils::run_log($cmd);
+ ok($result, "@$cmd exit code 0");
+ my $log =
+ PostgreSQL::Test::Utils::slurp_file($self->logfile, $log_location);
+ unlike($log, $prohibited_sql, "$test_name: SQL found in server log");
+ return;
+}
+
=item $node->log_content()
--
2.47.1
[text/x-patch] v37-0011-Add-force-analyze-to-vacuumdb.patch (12.1K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/12-v37-0011-Add-force-analyze-to-vacuumdb.patch)
download | inline diff:
From 091e1f07f758c110536c6a53d2f70f892212ed18 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Fri, 8 Nov 2024 12:27:50 -0500
Subject: [PATCH v37 11/11] Add --force-analyze to vacuumdb.
The vacuumdb options of --analyze-in-stages and --analyze-only are often
used after a restore from a dump or a pg_upgrade to quickly rebuild
stats on a databse.
However, now that stats are imported in most (but not all) cases,
running either of these commands will be at least partially redundant,
and will overwrite the stats that were just imported, which is a big
POLA violation.
We could add a new option such as --analyze-missing-in-stages, but that
wouldn't help the userbase that grown accustomed to running
--analyze-in-stages after an upgrade.
The least-bad option to handle the situation is to change the behavior
of --analyze-only and --analyze-in-stages to only analyze tables which
were missing stats before the vacuumdb started, but offer the
--force-analyze flag to restore the old behavior for those who truly
wanted it.
---
src/bin/scripts/t/100_vacuumdb.pl | 6 +-
src/bin/scripts/vacuumdb.c | 91 ++++++++++++++++++++++++-------
doc/src/sgml/ref/vacuumdb.sgml | 48 ++++++++++++++++
3 files changed, 125 insertions(+), 20 deletions(-)
diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl
index ccb7711af4..a3fcfda5ee 100644
--- a/src/bin/scripts/t/100_vacuumdb.pl
+++ b/src/bin/scripts/t/100_vacuumdb.pl
@@ -127,9 +127,13 @@ $node->issues_sql_like(
qr/statement: VACUUM \(SKIP_DATABASE_STATS, ANALYZE\) public.vactable\(a, b\);/,
'vacuumdb --analyze with complete column list');
$node->issues_sql_like(
+ [ 'vacuumdb', '--analyze-only', '--force-analyze', '--table', 'vactable(b)', 'postgres' ],
+ qr/statement: ANALYZE public.vactable\(b\);/,
+ 'vacuumdb --analyze-only --force-analyze with partial column list');
+$node->issues_sql_unlike(
[ 'vacuumdb', '--analyze-only', '--table', 'vactable(b)', 'postgres' ],
qr/statement: ANALYZE public.vactable\(b\);/,
- 'vacuumdb --analyze-only with partial column list');
+ 'vacuumdb --analyze-only --force-analyze with partial column list skipping vacuumed tables');
$node->command_checks_all(
[ 'vacuumdb', '--analyze', '--table', 'vacview', 'postgres' ],
0,
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 28506ca0c2..30804af75d 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -25,6 +25,7 @@
#include "fe_utils/query_utils.h"
#include "fe_utils/simple_list.h"
#include "fe_utils/string_utils.h"
+#include "pqexpbuffer.h"
/* vacuum options controlled by user flags */
@@ -47,6 +48,8 @@ typedef struct vacuumingOptions
bool process_main;
bool process_toast;
bool skip_database_stats;
+ bool analyze_in_stages;
+ bool force_analyze;
char *buffer_usage_limit;
} vacuumingOptions;
@@ -75,7 +78,6 @@ static void vacuum_one_database(ConnParams *cparams,
static void vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet);
@@ -140,6 +142,7 @@ main(int argc, char *argv[])
{"no-process-toast", no_argument, NULL, 11},
{"no-process-main", no_argument, NULL, 12},
{"buffer-usage-limit", required_argument, NULL, 13},
+ {"force-analyze", no_argument, NULL, 14},
{NULL, 0, NULL, 0}
};
@@ -156,7 +159,6 @@ main(int argc, char *argv[])
bool echo = false;
bool quiet = false;
vacuumingOptions vacopts;
- bool analyze_in_stages = false;
SimpleStringList objects = {NULL, NULL};
int concurrentCons = 1;
int tbl_count = 0;
@@ -170,6 +172,8 @@ main(int argc, char *argv[])
vacopts.do_truncate = true;
vacopts.process_main = true;
vacopts.process_toast = true;
+ vacopts.force_analyze = false;
+ vacopts.analyze_in_stages = false;
pg_logging_init(argv[0]);
progname = get_progname(argv[0]);
@@ -251,7 +255,7 @@ main(int argc, char *argv[])
maintenance_db = pg_strdup(optarg);
break;
case 3:
- analyze_in_stages = vacopts.analyze_only = true;
+ vacopts.analyze_in_stages = vacopts.analyze_only = true;
break;
case 4:
vacopts.disable_page_skipping = true;
@@ -287,6 +291,9 @@ main(int argc, char *argv[])
case 13:
vacopts.buffer_usage_limit = escape_quotes(optarg);
break;
+ case 14:
+ vacopts.force_analyze = true;
+ break;
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -372,6 +379,14 @@ main(int argc, char *argv[])
pg_fatal("cannot use the \"%s\" option with the \"%s\" option",
"buffer-usage-limit", "full");
+ /*
+ * --force-analyze is only valid when used with --analyze-only, -analyze,
+ * or --analyze-in-stages
+ */
+ if (vacopts.force_analyze && !vacopts.analyze_only && !vacopts.analyze_in_stages)
+ pg_fatal("can only use the \"%s\" option with \"%s\" or \"%s\"",
+ "--force-analyze", "-Z/--analyze-only", "--analyze-in-stages");
+
/* fill cparams except for dbname, which is set below */
cparams.pghost = host;
cparams.pgport = port;
@@ -390,7 +405,6 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
vacuum_all_databases(&cparams, &vacopts,
- analyze_in_stages,
&objects,
concurrentCons,
progname, echo, quiet);
@@ -413,20 +427,27 @@ main(int argc, char *argv[])
}
cparams.dbname = dbname;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts.analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectDatabase(&cparams, progname, echo, false, true);
check_conn_options(conn, &vacopts);
print_processing_notice(conn, stage, progname, quiet);
found_objects = generate_catalog_list(conn, &vacopts, &objects, echo, &ntup);
+ vacuum_one_database(&cparams, &vacopts, stage,
+ &objects,
+ concurrentCons,
+ conn,
+ found_objects,
+ ntup,
+ progname, echo, quiet);
- if (analyze_in_stages)
+ if (stage != ANALYZE_NO_STAGE)
{
- for (stage = 0; stage < ANALYZE_NUM_STAGES; stage++)
+ for (stage = 1; stage < ANALYZE_NUM_STAGES; stage++)
{
/* the last pass disconnected the conn */
- if (stage > 0)
- conn = connectDatabase(&cparams, progname, echo, false, true);
+ conn = connectDatabase(&cparams, progname, echo, false, true);
+ print_processing_notice(conn, stage, progname, quiet);
vacuum_one_database(&cparams, &vacopts, stage,
&objects,
@@ -437,14 +458,6 @@ main(int argc, char *argv[])
progname, echo, quiet);
}
}
- else
- vacuum_one_database(&cparams, &vacopts, stage,
- &objects,
- concurrentCons,
- conn,
- found_objects,
- ntup,
- progname, echo, quiet);
}
exit(0);
@@ -763,6 +776,47 @@ generate_catalog_list(PGconn *conn,
vacopts->min_mxid_age);
}
+ /*
+ * If this query is for an analyze-only or analyze-in-stages, two
+ * upgrade-centric operations, and force-analyze is NOT set, then
+ * exclude any relations that already have their full compliment
+ * of attribute stats and extended stats.
+ *
+ *
+ */
+ if ((vacopts->analyze_only || vacopts->analyze_in_stages) &&
+ !vacopts->force_analyze)
+ {
+ /*
+ * The pg_class in question has no pg_statistic rows representing
+ * user-visible columns that lack a corresponding pg_statitic row.
+ * Currently no differentiation is made for whether the
+ * pg_statistic.stainherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_attribute AS a\n"
+ " WHERE a.attrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND a.attnum OPERATOR(pg_catalog.>) 0 AND NOT a.attisdropped\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic AS s\n"
+ " WHERE s.starelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND s.staattnum OPERATOR(pg_catalog.=) a.attnum))\n");
+
+ /*
+ * The pg_class entry has no pg_statistic_ext rows that lack a corresponding
+ * pg_statistic_ext_data row. Currently no differentiation is made for whether
+ * pg_statistic_exta_data.stxdinherit is true or false.
+ */
+ appendPQExpBufferStr(&catalog_query,
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext AS e\n"
+ " WHERE e.stxrelid OPERATOR(pg_catalog.=) c.oid\n"
+ " AND NOT EXISTS (\n"
+ " SELECT NULL FROM pg_catalog.pg_statistic_ext_data AS d\n"
+ " WHERE d.stxoid OPERATOR(pg_catalog.=) e.oid))\n");
+ }
+
/*
* Execute the catalog query. We use the default search_path for this
* query for consistency with table lookups done elsewhere by the user.
@@ -959,7 +1013,6 @@ finish:
static void
vacuum_all_databases(ConnParams *cparams,
vacuumingOptions *vacopts,
- bool analyze_in_stages,
SimpleStringList *objects,
int concurrentCons,
const char *progname, bool echo, bool quiet)
@@ -972,7 +1025,7 @@ vacuum_all_databases(ConnParams *cparams,
SimpleStringList **found_objects;
int *num_tuples;
- stage = (analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
+ stage = (vacopts->analyze_in_stages) ? 0 : ANALYZE_NO_STAGE;
conn = connectMaintenanceDatabase(cparams, progname, echo);
result = executeQuery(conn,
diff --git a/doc/src/sgml/ref/vacuumdb.sgml b/doc/src/sgml/ref/vacuumdb.sgml
index 66fccb30a2..00ec927606 100644
--- a/doc/src/sgml/ref/vacuumdb.sgml
+++ b/doc/src/sgml/ref/vacuumdb.sgml
@@ -425,6 +425,12 @@ PostgreSQL documentation
<para>
Only calculate statistics for use by the optimizer (no vacuum).
</para>
+ <para>
+ By default, this operation excludes relations that already have
+ statistics generated. If the option <option>--force-analyze</option>
+ is also specified, then relations with existing stastistics are not
+ excluded.
+ </para>
</listitem>
</varlistentry>
@@ -439,6 +445,47 @@ PostgreSQL documentation
to produce usable statistics faster, and subsequent stages build the
full statistics.
</para>
+ <para>
+ This option is intended to be run after a <command>pg_upgrade</command>
+ to generate statistics for relations that have no stistatics or incomplete
+ statistics (such as those with extended statistics objects, which are not
+ imported on upgrade).
+ </para>
+ <para>
+ If the option <option>--force-analyze</option> is also specified, then
+ relations with existing stastistics are not excluded.
+ This option is only useful to analyze a database that currently has
+ no statistics or has wholly incorrect ones, such as if it is newly
+ populated from a restored dump.
+ Be aware that running with this option combinationin a database with
+ existing statistics may cause the query optimizer choices to become
+ transiently worse due to the low statistics targets of the early
+ stages.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--force-analyze</option></term>
+ <listitem>
+ <para>
+ This option can only be used if either <option>--analyze-only</option>
+ or <option>--analyze-in-stages</option> is specified. It modifies those
+ options to not filter out relations that already have statistics.
+ </para>
+ <para>
+
+ Only calculate statistics for use by the optimizer (no vacuum),
+ like <option>--analyze-only</option>. Run three
+ stages of analyze; the first stage uses the lowest possible statistics
+ target (see <xref linkend="guc-default-statistics-target"/>)
+ to produce usable statistics faster, and subsequent stages build the
+ full statistics.
+ </para>
+
+ <para>
+ This option was created
+ </para>
<para>
This option is only useful to analyze a database that currently has
@@ -452,6 +499,7 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+
<varlistentry>
<term><option>-?</option></term>
<term><option>--help</option></term>
--
2.47.1
[text/x-patch] v37-0008-split-out-generate_catalog_list.patch (7.1K, ../../CADkLM=c6NHdXU+d+m1yASZ4NT_qye1LCnaU2Vgf8YJ80jJT-Qg@mail.gmail.com/13-v37-0008-split-out-generate_catalog_list.patch)
download | inline diff:
From ceacfd5372fad6ad56780e0ac7ba129df5f72c36 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Wed, 6 Nov 2024 16:15:16 -0500
Subject: [PATCH v37 08/11] split out generate_catalog_list
---
src/bin/scripts/vacuumdb.c | 176 +++++++++++++++++++++----------------
1 file changed, 99 insertions(+), 77 deletions(-)
diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c
index 2aa55f191e..5ae7241716 100644
--- a/src/bin/scripts/vacuumdb.c
+++ b/src/bin/scripts/vacuumdb.c
@@ -560,67 +560,38 @@ print_processing_notice(PGconn *conn, int stage, const char *progname, bool quie
}
/*
- * vacuum_one_database
- *
- * Process tables in the given database. If the 'tables' list is empty,
- * process all tables in the database.
- *
- * Note that this function is only concerned with running exactly one stage
- * when in analyze-in-stages mode; caller must iterate on us if necessary.
- *
- * If concurrentCons is > 1, multiple connections are used to vacuum tables
- * in parallel. In this case and if the table list is empty, we first obtain
- * a list of tables from the database.
- */
-static void
-vacuum_one_database(ConnParams *cparams,
- vacuumingOptions *vacopts,
- int stage,
- SimpleStringList *objects,
- int concurrentCons,
- const char *progname, bool echo, bool quiet)
+ * Prepare the list of tables to process by querying the catalogs.
+ *
+ * Since we execute the constructed query with the default search_path
+ * (which could be unsafe), everything in this query MUST be fully
+ * qualified.
+ *
+ * First, build a WITH clause for the catalog query if any tables were
+ * specified, with a set of values made of relation names and their
+ * optional set of columns. This is used to match any provided column
+ * lists with the generated qualified identifiers and to filter for the
+ * tables provided via --table. If a listed table does not exist, the
+ * catalog query will fail.
+ */
+static SimpleStringList *
+generate_catalog_list(PGconn *conn,
+ vacuumingOptions *vacopts,
+ SimpleStringList *objects,
+ bool echo,
+ int *ntups)
{
- PQExpBufferData sql;
- PQExpBufferData buf;
PQExpBufferData catalog_query;
- PGresult *res;
- PGconn *conn;
+ PQExpBufferData buf;
+ SimpleStringList *dbtables;
SimpleStringListCell *cell;
- ParallelSlotArray *sa;
- SimpleStringList dbtables = {NULL, NULL};
- int i;
- int ntups;
- bool failed = false;
bool objects_listed = false;
- const char *initcmd;
- const char *stage_commands[] = {
- "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
- "SET default_statistics_target=10; RESET vacuum_cost_delay;",
- "RESET default_statistics_target;"
- };
+ PGresult *res;
+ int i;
- Assert(stage == ANALYZE_NO_STAGE ||
- (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+ dbtables = palloc(sizeof(SimpleStringList));
+ dbtables->head = NULL;
+ dbtables->tail = NULL;
- conn = connectDatabase(cparams, progname, echo, false, true);
-
- check_conn_options(conn, vacopts);
- print_processing_notice(conn, stage, progname, quiet);
-
- /*
- * Prepare the list of tables to process by querying the catalogs.
- *
- * Since we execute the constructed query with the default search_path
- * (which could be unsafe), everything in this query MUST be fully
- * qualified.
- *
- * First, build a WITH clause for the catalog query if any tables were
- * specified, with a set of values made of relation names and their
- * optional set of columns. This is used to match any provided column
- * lists with the generated qualified identifiers and to filter for the
- * tables provided via --table. If a listed table does not exist, the
- * catalog query will fail.
- */
initPQExpBuffer(&catalog_query);
for (cell = objects ? objects->head : NULL; cell; cell = cell->next)
{
@@ -771,40 +742,91 @@ vacuum_one_database(ConnParams *cparams,
appendPQExpBufferStr(&catalog_query, " ORDER BY c.relpages DESC;");
executeCommand(conn, "RESET search_path;", echo);
res = executeQuery(conn, catalog_query.data, echo);
+ *ntups = PQntuples(res);
termPQExpBuffer(&catalog_query);
PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo));
- /*
- * If no rows are returned, there are no matching tables, so we are done.
- */
- ntups = PQntuples(res);
- if (ntups == 0)
- {
- PQclear(res);
- PQfinish(conn);
- return;
- }
-
/*
* Build qualified identifiers for each table, including the column list
* if given.
*/
- initPQExpBuffer(&buf);
- for (i = 0; i < ntups; i++)
+ if (*ntups > 0)
{
- appendPQExpBufferStr(&buf,
- fmtQualifiedId(PQgetvalue(res, i, 1),
- PQgetvalue(res, i, 0)));
+ initPQExpBuffer(&buf);
+ for (i = 0; i < *ntups; i++)
+ {
+ appendPQExpBufferStr(&buf,
+ fmtQualifiedId(PQgetvalue(res, i, 1),
+ PQgetvalue(res, i, 0)));
- if (objects_listed && !PQgetisnull(res, i, 2))
- appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
+ if (objects_listed && !PQgetisnull(res, i, 2))
+ appendPQExpBufferStr(&buf, PQgetvalue(res, i, 2));
- simple_string_list_append(&dbtables, buf.data);
- resetPQExpBuffer(&buf);
+ simple_string_list_append(dbtables, buf.data);
+ resetPQExpBuffer(&buf);
+ }
+ termPQExpBuffer(&buf);
}
- termPQExpBuffer(&buf);
PQclear(res);
+ return dbtables;
+}
+
+/*
+ * vacuum_one_database
+ *
+ * Process tables in the given database. If the 'tables' list is empty,
+ * process all tables in the database.
+ *
+ * Note that this function is only concerned with running exactly one stage
+ * when in analyze-in-stages mode; caller must iterate on us if necessary.
+ *
+ * If concurrentCons is > 1, multiple connections are used to vacuum tables
+ * in parallel. In this case and if the table list is empty, we first obtain
+ * a list of tables from the database.
+ */
+static void
+vacuum_one_database(ConnParams *cparams,
+ vacuumingOptions *vacopts,
+ int stage,
+ SimpleStringList *objects,
+ int concurrentCons,
+ const char *progname, bool echo, bool quiet)
+{
+ PQExpBufferData sql;
+ PGconn *conn;
+ SimpleStringListCell *cell;
+ ParallelSlotArray *sa;
+ int ntups;
+ bool failed = false;
+ const char *initcmd;
+ SimpleStringList *dbtables;
+ const char *stage_commands[] = {
+ "SET default_statistics_target=1; SET vacuum_cost_delay=0;",
+ "SET default_statistics_target=10; RESET vacuum_cost_delay;",
+ "RESET default_statistics_target;"
+ };
+
+ Assert(stage == ANALYZE_NO_STAGE ||
+ (stage >= 0 && stage < ANALYZE_NUM_STAGES));
+
+ conn = connectDatabase(cparams, progname, echo, false, true);
+
+ check_conn_options(conn, vacopts);
+ print_processing_notice(conn, stage, progname, quiet);
+
+ dbtables = generate_catalog_list(conn, vacopts, objects, echo, &ntups);
+
+ /*
+ * If no rows are returned, there are no matching tables, so we are done.
+ */
+ if (ntups == 0)
+ {
+ PQfinish(conn);
+ return;
+ }
+
+
/*
* Ensure concurrentCons is sane. If there are more connections than
* vacuumable relations, we don't need to use them all.
@@ -837,7 +859,7 @@ vacuum_one_database(ConnParams *cparams,
initPQExpBuffer(&sql);
- cell = dbtables.head;
+ cell = dbtables->head;
do
{
const char *tabname = cell->val;
--
2.47.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-01-07 20:14 ` Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2025-01-07 20:14 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; Nathan Bossart <[email protected]>; +Cc: Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Tue, 2025-01-07 at 01:18 -0500, Corey Huinker wrote:
> Attached is the latest (and probably last) unified patchset before
> parts get spun off into their own threads.
In this thread I'm only looking at 0001. Please start a new thread for
vacuumdb and extended stats changes.
> 0001 - This is the unified changes to pg_dump, pg_restore,
> pg_dumpall, and pg_upgrade.
>
> It incorporates most of what Jeff changed when he unified v36j, with
> typo fixes spotted by Bruce. There was interest in splitting
> STATISTICS DATA into RELATION STATISTICS DATA and ATTRIBUTE
> STATISTICS DATA.
I think we should just stick with "STATISTICS DATA".
> There was also interest in changing the prefix for STATISTICS DATA.
> However, the only special case for prefixes currently relies on an
> isData flag. Since there is no isStatistics flag, we would either
> have to create one, or do strcmps on te->description looking for
> "STATISTICS DATA". It's do-able, but I'm not sure it's worth it.
I do like the idea of a "Statistics for ..." prefix, and I think it's
doable.
The caller needs some knowledge about that anyway, to correctly output
the statistics dump when the schema is not requested. Tests should
cover those cases, too.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-01-15 20:56 ` Corey Huinker <[email protected]>
2025-01-16 14:43 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
0 siblings, 2 replies; 81+ messages in thread
From: Corey Huinker @ 2025-01-15 20:56 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; jian he <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
>
> I do like the idea of a "Statistics for ..." prefix, and I think it's
> doable.
>
And that's now implemented.
The caller needs some knowledge about that anyway, to correctly output
> the statistics dump when the schema is not requested. Tests should
> cover those cases, too.
>
Tests for pg_dump --no-statistics and pg_dump --schema-only were added.
Rebased to master as of today.
I'm not completely happy with this patch, as I had to comment out one check
in pg_backup_archiver that seemed necessary, but perhaps another set of
eyes will set me straight.
Attached is just the pg_dump stuff, and only for relation/attribute stats.
The extended stats and vacuumdb work will be in their own threads going
forward.
Attachments:
[text/x-patch] v38-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch (61.7K, ../../CADkLM=fe4f=YVGHbOGiXV+W2SoJZtcNw1Ta8gYh5UUhxndaVgw@mail.gmail.com/3-v38-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From fdb53409458bc9aeed3496f355173ac97062afd1 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v38] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute. These
statements will restore the statistics of the current system onto the
destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, staistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index and Materialized View
statistics are dumped in the post-data section.
Add --no-data option.
This option is useful for situations where someone wishes to test
query plans from a production database without copying production data.
This also makes the corresponding change to the simulated pg_upgrade in
the TAP tests for pg_dump.
This checks that dumping statistics is now the default, and that
--no-statistics will suppress statistics.
Add --no-schema option to pg_dump, etc.
Previously, users could use --data-only when they wanted to suppress
schema from a dump. However, that no longer makes sense now that the
data/schema binary has become the data/schema/statistics trinary.
---
src/bin/pg_dump/pg_backup.h | 10 +-
src/bin/pg_dump/pg_backup_archiver.c | 70 ++++-
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_dump.c | 384 ++++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 9 +
src/bin/pg_dump/pg_dump_sort.c | 32 ++-
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 32 ++-
src/bin/pg_dump/t/001_basic.pl | 18 ++
src/bin/pg_dump/t/002_pg_dump.pl | 61 ++++-
src/bin/pg_upgrade/dump.c | 6 +-
src/bin/pg_upgrade/option.c | 12 +
src/bin/pg_upgrade/pg_upgrade.h | 1 +
doc/src/sgml/ref/pg_dump.sgml | 69 +++--
doc/src/sgml/ref/pg_dumpall.sgml | 38 +++
doc/src/sgml/ref/pg_restore.sgml | 51 +++-
doc/src/sgml/ref/pgupgrade.sgml | 18 ++
src/tools/pgindent/typedefs.list | 1 +
18 files changed, 771 insertions(+), 49 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b2..3fa1474fad 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,9 +110,12 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +163,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -179,8 +183,11 @@ typedef struct _dumpOptions
int column_inserts;
int if_exists;
int no_comments;
- int no_security_labels;
+ int no_data;
int no_publications;
+ int no_schema;
+ int no_security_labels;
+ int no_statistics;
int no_subscriptions;
int no_toast_compression;
int no_unlogged_table_data;
@@ -208,6 +215,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844..d651b9b764 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -46,6 +46,11 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
+typedef enum entryType {
+ default_entry,
+ data_entry,
+ statistics_entry
+} entryType;
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -53,7 +58,7 @@ static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr,
DataDirSyncMethod sync_method);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
-static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
+static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type);
static char *sanitize_line(const char *str, bool want_hyphen);
static void _doSetFixedOutputState(ArchiveHandle *AH);
static void _doSetSessionAuth(ArchiveHandle *AH, const char *user);
@@ -149,6 +154,7 @@ InitDumpOptions(DumpOptions *opts)
opts->dumpSections = DUMP_UNSECTIONED;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
}
/*
@@ -169,9 +175,10 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->outputClean = ropt->dropSchema;
dopt->dumpData = ropt->dumpData;
dopt->dumpSchema = ropt->dumpSchema;
+ dopt->dumpSections = ropt->dumpSections;
+ dopt->dumpStatistics = ropt->dumpStatistics;
dopt->if_exists = ropt->if_exists;
dopt->column_inserts = ropt->column_inserts;
- dopt->dumpSections = ropt->dumpSections;
dopt->aclsSkip = ropt->aclsSkip;
dopt->outputSuperuser = ropt->superuser;
dopt->outputCreateDB = ropt->createDB;
@@ -186,6 +193,9 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
@@ -739,7 +749,7 @@ RestoreArchive(Archive *AHX)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
continue; /* ignore if not to be dumped at all */
switch (_tocEntryRestorePass(te))
@@ -760,7 +770,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -770,7 +780,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_POST_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -869,7 +879,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
pg_log_info("creating %s \"%s\"",
te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
defnDumped = true;
if (strcmp(te->desc, "TABLE") == 0)
@@ -938,7 +948,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
*/
if (AH->PrintTocDataPtr != NULL)
{
- _printTocEntry(AH, te, true);
+ _printTocEntry(AH, te, data_entry);
if (strcmp(te->desc, "BLOBS") == 0 ||
strcmp(te->desc, "BLOB COMMENTS") == 0)
@@ -1036,15 +1046,24 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
{
/* If we haven't already dumped the defn part, do so now */
pg_log_info("executing %s %s", te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
}
}
+ /*
+ * If it has a statistics component that we want, then process that
+ */
+ if ((reqs & REQ_STATS) != 0)
+ {
+ _printTocEntry(AH, te, statistics_entry);
+ defnDumped = true;
+ }
+
/*
* If we emitted anything for this TOC entry, that counts as one action
* against the transaction-size limit. Commit if it's time to.
*/
- if ((reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && ropt->txn_size > 0)
+ if ((reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && ropt->txn_size > 0)
{
if (++AH->txnCount >= ropt->txn_size)
{
@@ -1084,6 +1103,7 @@ NewRestoreOptions(void)
opts->compression_spec.level = 0;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
return opts;
}
@@ -1093,6 +1113,7 @@ _disableTriggersIfNecessary(ArchiveHandle *AH, TocEntry *te)
{
RestoreOptions *ropt = AH->public.ropt;
+ ahprintf(AH, "-- ALTER TABLE DISABLE TRIGGER ALL %d %d; \n\n", ropt->dumpSchema, ropt->disable_triggers);
/* This hack is only needed in a data-only restore */
if (ropt->dumpSchema || !ropt->disable_triggers)
return;
@@ -2962,6 +2983,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's statistics and we don't want statistics, maybe ignore it */
+ if (!ropt->dumpStatistics && strcmp(te->desc, "STATISTICS DATA") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +3016,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS DATA") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
@@ -3107,6 +3133,15 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
}
+ /*
+ * Statistics Data entries have no other components.
+ */
+ /*
+ * TODO: removed for now
+ if (strcmp(te->desc, "STATISTICS DATA") == 0)
+ return REQ_STATS;
+ */
+
/*
* Determine whether the TOC entry contains schema and/or data components,
* and mask off inapplicable REQ bits. If it had a dataDumper, assume
@@ -3729,7 +3764,7 @@ _getObjectDescription(PQExpBuffer buf, const TocEntry *te)
* will remain at default, until the matching ACL TOC entry is restored.
*/
static void
-_printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
+_printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type)
{
RestoreOptions *ropt = AH->public.ropt;
@@ -3753,10 +3788,17 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
char *sanitized_schema;
char *sanitized_owner;
- if (isData)
- pfx = "Data for ";
- else
- pfx = "";
+ switch (entry_type)
+ {
+ case data_entry:
+ pfx = "Data for ";
+ break;
+ case statistics_entry:
+ pfx = "Statistics for ";
+ break;
+ default:
+ pfx = "";
+ }
ahprintf(AH, "--\n");
if (AH->public.verbose)
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index ce5ed1dd39..a2064f471e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -209,7 +209,8 @@ typedef enum
#define REQ_SCHEMA 0x01 /* want schema */
#define REQ_DATA 0x02 /* want data */
-#define REQ_SPECIAL 0x04 /* for special TOC entries */
+#define REQ_STATS 0x04
+#define REQ_SPECIAL 0x08 /* for special TOC entries */
struct _archiveHandle
{
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 8f73a5df95..9e7cb2c48f 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -430,6 +430,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -466,6 +467,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -492,8 +494,11 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -539,7 +544,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:PRsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -613,6 +618,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -784,6 +793,17 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -798,8 +818,9 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
@@ -1099,6 +1120,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1177,7 +1199,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1190,11 +1212,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1219,8 +1242,11 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
+ printf(_(" --no-schema do not dump schema\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6777,6 +6803,43 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+ info->postponed_def = false;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7154,6 +7217,7 @@ getTables(Archive *fout, int *numTables)
/* Tables have data */
tblinfo[i].dobj.components |= DUMP_COMPONENT_DATA;
+ tblinfo[i].dobj.components |= DUMP_COMPONENT_STATISTICS;
/* Mark whether table has an ACL */
if (!PQgetisnull(res, i, i_relacl))
@@ -7202,6 +7266,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7648,11 +7714,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7675,7 +7744,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7709,6 +7785,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10296,6 +10374,287 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * Decide which section to use based on the relkind of the parent object.
+ *
+ * NB: materialized views may be postponed from SECTION_PRE_DATA to
+ * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the
+ * matview stats will also be postponed to SECTION_POST_DATA. See
+ * repairMatViewBoundaryMultiLoop().
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+ switch (rsinfo->relkind)
+ {
+ case RELKIND_RELATION:
+ case RELKIND_PARTITIONED_TABLE:
+ case RELKIND_MATVIEW:
+ return SECTION_DATA;
+ case RELKIND_INDEX:
+ case RELKIND_PARTITIONED_INDEX:
+ return SECTION_POST_DATA;
+ default:
+ pg_fatal("cannot dump statistics for relation kind '%c'",
+ rsinfo->relkind);
+ }
+
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = rsinfo->postponed_def ?
+ SECTION_POST_DATA : statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = dobj->dependencies,
+ .nDeps = dobj->nDeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10744,6 +11103,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -17183,6 +17545,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo)
free(indstatvalsarray);
}
+ /* Comments and stats share same .dep */
+
/* Dump Index Comments */
if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT)
dumpComment(fout, "INDEX", qindxname,
@@ -18970,6 +19334,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index f62b564ed1..4edd88a54b 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -84,6 +84,7 @@ typedef enum
DO_PUBLICATION_TABLE_IN_SCHEMA,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
+ DO_REL_STATS,
} DumpableObjectType;
/*
@@ -109,6 +110,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -429,6 +431,13 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'v', 'c', etc */
+ bool postponed_def;
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index dc9a28924b..3a3602e3d2 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -801,11 +801,21 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj,
{
/* remove boundary's dependency on object after it in loop */
removeObjectDependency(boundaryobj, nextobj->dumpId);
- /* if that object is a matview, mark it as postponed into post-data */
+ /*
+ * If that object is a matview or matview status, mark it as postponed into
+ * post-data.
+ */
if (nextobj->objType == DO_TABLE)
{
TableInfo *nextinfo = (TableInfo *) nextobj;
+ if (nextinfo->relkind == RELKIND_MATVIEW)
+ nextinfo->postponed_def = true;
+ }
+ else if (nextobj->objType == DO_REL_STATS)
+ {
+ RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj;
+
if (nextinfo->relkind == RELKIND_MATVIEW)
nextinfo->postponed_def = true;
}
@@ -1018,6 +1028,21 @@ repairDependencyLoop(DumpableObject **loop,
{
DumpableObject *nextobj;
+ nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
+ repairMatViewBoundaryMultiLoop(loop[j], nextobj);
+ return;
+ }
+ }
+ }
+ else if (loop[i]->objType == DO_REL_STATS &&
+ ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW)
+ {
+ for (j = 0; j < nLoop; j++)
+ {
+ if (loop[j]->objType == DO_POST_DATA_BOUNDARY)
+ {
+ DumpableObject *nextobj;
+
nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
repairMatViewBoundaryMultiLoop(loop[j], nextobj);
return;
@@ -1500,6 +1525,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 396f79781c..7effb70490 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 88ae39d938..9586bd032c 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,9 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool data_only = false;
+ bool schema_only = false;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -71,12 +74,13 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_comments = 0;
+ static int no_data = 0;
static int no_publications = 0;
+ static int no_schema = 0;
static int no_security_labels = 0;
+ static int no_statistics = 0;
static int no_subscriptions = 0;
static int strict_names = 0;
- bool data_only = false;
- bool schema_only = false;
struct option cmdopts[] = {
{"clean", 0, NULL, 'c'},
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'P'},
/*
* the following options don't have an equivalent short option letter
@@ -124,9 +129,12 @@ main(int argc, char **argv)
{"transaction-size", required_argument, NULL, 5},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
@@ -271,6 +279,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +355,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +378,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpData = data_only || (!no_data && !schema_only && !statistics_only);
+ opts->dumpSchema = schema_only || (!no_schema && !data_only && !statistics_only);
+ opts->dumpStatistics = statistics_only || (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +392,8 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
@@ -484,6 +503,7 @@ usage(const char *progname)
printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n"));
printf(_(" -T, --trigger=NAME restore named trigger\n"));
printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only restore only the statistics, not schema or data\n"));
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
@@ -491,10 +511,14 @@ usage(const char *progname)
" in FILENAME\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comment commands\n"));
+ printf(_(" --no-data do not restore data\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
+ /* This hack is only needed in a data-only restore */
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 214240f1ae..f29da06ed2 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index bf65d44b94..737b184ea9 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -66,7 +66,7 @@ my %pgdump_runs = (
'--format=custom',
"--file=$tempdir/binary_upgrade.dump",
'-w',
- '--schema-only',
+ '--no-data',
'--binary-upgrade',
'-d', 'postgres', # alternative way to specify database
],
@@ -645,7 +645,19 @@ my %pgdump_runs = (
'--schema=dump_test', '-b', '-B', '--no-sync', 'postgres',
],
- },);
+ },
+ no_statistics => {
+ dump_cmd => [
+ 'pg_dump', "--file=$tempdir/no_statistics.sql",
+ '--no-sync', '--no-statistics', 'postgres',
+ ],
+ },
+ no_schema => {
+ dump_cmd => [
+ 'pg_dump', "--file=$tempdir/no_schema.sql",
+ '--no-sync', '--no-schema', 'postgres',
+ ],
+ });
###############################################################
# Definition of the tests to run.
@@ -711,6 +723,7 @@ my %full_runs = (
no_large_objects => 1,
no_owner => 1,
no_privs => 1,
+ no_statistics => 1,
no_table_access_method => 1,
pg_dumpall_dbprivs => 1,
pg_dumpall_exclude => 1,
@@ -912,6 +925,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1325,6 +1339,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1346,6 +1361,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1367,6 +1383,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1533,6 +1550,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1686,6 +1704,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_test_table => 1,
section_data => 1,
},
@@ -1713,6 +1732,7 @@ my %tests = (
data_only => 1,
exclude_test_table => 1,
exclude_test_table_data => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1733,7 +1753,10 @@ my %tests = (
\QCOPY dump_test.fk_reference_test_table (col1) FROM stdin;\E
\n(?:\d\n){5}\\\.\n
/xms,
- like => { data_only => 1, },
+ like => {
+ data_only => 1,
+ no_schema => 1,
+ },
},
'COPY test_second_table' => {
@@ -1749,6 +1772,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1771,6 +1795,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1794,6 +1819,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1816,6 +1842,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1838,6 +1865,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -3234,6 +3262,7 @@ my %tests = (
like => {
%full_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
only_dump_test_schema => 1,
test_schema_plus_large_objects => 1,
@@ -3404,6 +3433,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_measurement => 1,
section_data => 1,
only_dump_test_schema => 1,
@@ -4286,6 +4316,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
binary_upgrade => 1,
@@ -4586,6 +4617,30 @@ my %tests = (
},
},
+ #
+ # Table statistics should go in section=data.
+ # Materialized view statistics should go in section=post-data.
+ #
+ 'statistics_import' => {
+ create_sql => '
+ CREATE TABLE dump_test.has_stats
+ AS SELECT g.g AS x, g.g / 2 AS y FROM generate_series(1,100) AS g(g);
+ CREATE TABLE dump_test.has_stats_mv AS SELECT * FROM dump_test.has_stats;
+ ANALYZE dump_test.has_stats, dump_test.has_stats_mv;',
+ regexp => qr/pg_catalog.pg_restore_attribute_stats/,
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ section_data => 1,
+ },
+ unlike => {
+ exclude_dump_test_schema => 1,
+ no_statistics => 1,
+ only_dump_measurement => 1,
+ schema_only => 1,
+ },
+ },
+
# CREATE TABLE with partitioned table and various AMs. One
# partition uses the same default as the parent, and a second
# uses its own AM.
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8ce0fa3020..a29cd2cca9 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 108eb7a1ba..3b6c7ec994 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -306,7 +316,9 @@ usage(void)
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
+ printf(_(" --no-statistics do not import statistics from old cluster\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statistics import statistics from old cluster (default)\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 0cdd675e4f..3fe111fbde 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index d66e901f51..5e58f24d21 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,13 +141,11 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
- switch is therefore only useful to add large objects to dumps
- where a specific schema or table has been requested. Note that
- large objects are considered data and therefore will be included when
- <option>--data-only</option> is used, but not
- when <option>--schema-only</option> is.
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or <option>--statistics-only</option>, or
+ <option>--no-data</option> is specified. The <option>-b</option>
+ switch is therefore only useful to add large objects to dumps where a
+ specific schema or table has been requested.
</para>
</listitem>
</varlistentry>
@@ -516,10 +514,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +651,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -824,16 +835,17 @@ PostgreSQL documentation
<term><option>--exclude-table-data=<replaceable class="parameter">pattern</replaceable></option></term>
<listitem>
<para>
- Do not dump data for any tables matching <replaceable
+ Do not dump data or statistics for any tables matching <replaceable
class="parameter">pattern</replaceable>. The pattern is
interpreted according to the same rules as for <option>-t</option>.
<option>--exclude-table-data</option> can be given more than once to
- exclude tables matching any of several patterns. This option is
- useful when you need the definition of a particular table even
- though you do not need the data in it.
+ exclude tables matching any of several patterns. This option is useful
+ when you need the definition of a particular table even though you do
+ not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1080,6 +1092,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -1098,6 +1119,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index 014f279258..d423153a93 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -265,6 +265,17 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--binary-upgrade</option></term>
<listitem>
@@ -422,6 +433,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -447,6 +467,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -456,6 +485,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719..22c3c118ad 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,20 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema (data definitions) or data.
+ </para>
+ <para>
+ (Do not confuse this with the <option>--schema</option> option, which
+ uses the word <quote>schema</quote> in a different meaning.)
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -681,6 +696,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore data, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-data-for-failed-tables</option></term>
<listitem>
@@ -713,6 +738,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema (data definitions), even if
+ the archive contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -723,6 +758,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml
index 4777381dac..64a1ebd613 100644
--- a/doc/src/sgml/ref/pgupgrade.sgml
+++ b/doc/src/sgml/ref/pgupgrade.sgml
@@ -145,6 +145,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-statistics</option></term>
+ <listitem>
+ <para>
+ Restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-o</option> <replaceable class="parameter">options</replaceable></term>
<term><option>--old-options</option> <replaceable class="parameter">options</replaceable></term>
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 94dc956ae8..eab0d0f84c 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2397,6 +2397,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelStatsInfo
RelToCheck
RelToCluster
RelabelType
--
2.48.0
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-01-16 14:43 ` jian he <[email protected]>
2025-01-16 15:52 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: jian he @ 2025-01-16 14:43 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Thu, Jan 16, 2025 at 4:56 AM Corey Huinker <[email protected]> wrote:
>>
> Attached is just the pg_dump stuff, and only for relation/attribute stats. The extended stats and vacuumdb work will be in their own threads going forward.
I didn't follow this thread actively,
so if the following issue is already addressed, please forgive me.
pg_dump --dbname=src2 --table=tenk1 --statistics-only --verbose > x.sql
there no pg_restore_attribute_stats, pg_restore_relation_stats there
for table tenk1.
There aren't any statistics, should there be?
also should
pg_dump --dbname=src2 --no-data --no-schema --table=tenk1
dump the statistics? currently seems no.
in doc/src/sgml/ref/pg_dump.sgml
there are six options to control the main output now.
--schema-only, --statistics-only, --data-only,
--no-schema, --no-data, --no-statistics
maybe we need spare one paragraph to explain the default behavior,
and also have an example on it?
pg_dump --dbname=src2 --table=tenk1 --verbose > 1.sql
seems the statistics dump (pg_restore_attribute_stats) is ordered by
attribute name.
should it make more sense to order by attnum?
getRelationStatistics
typedef struct _relStatsInfo
{
DumpableObject dobj;
char relkind; /* 'r', 'v', 'c', etc */
bool postponed_def;
} RelStatsInfo;
comment /* 'r', 'v', 'c', etc */
Is it wrong? there is no relkind='c'.
field postponed_def really deserves a comment.
we also need change
enum dbObjectTypePriorities
static const int dbObjectTypePriority[]
?
in dumpRelationStats, we can add Assert on it.
if (!fout->dopt->dumpStatistics)
return;
Assert(dobj->dump & DUMP_COMPONENT_STATISTICS);
I found out the owner's info is missing in the dumped content.
for example, the line "Name: STATISTICS DATA tenk1_pkey;" missing owner info.
not sure this is intended?
--
-- Name: tenk1 tenk1_pkey; Type: CONSTRAINT; Schema: public; Owner: jian
--
ALTER TABLE ONLY public.tenk1
ADD CONSTRAINT tenk1_pkey PRIMARY KEY (unique1);
--
-- Name: STATISTICS DATA tenk1_pkey; Type: STATISTICS DATA; Schema:
public; Owner: -
--
SELECT * FROM pg_catalog.pg_restore_relation_stats(
'relation', 'public.tenk1_pkey'::regclass,
'version', '180000'::integer,
'relpages', '30'::integer,
'reltuples', '10000'::real,
'relallvisible', '0'::integer
);
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-16 14:43 ` Re: Statistics Import and Export jian he <[email protected]>
@ 2025-01-16 15:52 ` Corey Huinker <[email protected]>
2025-01-17 14:20 ` Re: Statistics Import and Export jian he <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-01-16 15:52 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
> pg_dump --dbname=src2 --table=tenk1 --statistics-only --verbose > x.sql
> there no pg_restore_attribute_stats, pg_restore_relation_stats there
> for table tenk1.
> There aren't any statistics, should there be?
>
pg_restore_relation_stats: yes. looking into that.
pg_restore_attribute_stats: yes IF the table had been analyzed, no
otherwise.
> also should
> pg_dump --dbname=src2 --no-data --no-schema --table=tenk1
> dump the statistics? currently seems no.
>
> in doc/src/sgml/ref/pg_dump.sgml
> there are six options to control the main output now.
> --schema-only, --statistics-only, --data-only,
> --no-schema, --no-data, --no-statistics
> maybe we need spare one paragraph to explain the default behavior,
> and also have an example on it?
>
+1
> pg_dump --dbname=src2 --table=tenk1 --verbose > 1.sql
> seems the statistics dump (pg_restore_attribute_stats) is ordered by
> attribute name.
> should it make more sense to order by attnum?
>
For security reasons, we pull attribute statistics from pg_stats (
https://www.postgresql.org/docs/current/view-pg-stats.html) which does not
have attnum. The only reason we order the list is to ensure that dump
comparison tests match.
> getRelationStatistics
> typedef struct _relStatsInfo
> {
> DumpableObject dobj;
> char relkind; /* 'r', 'v', 'c', etc */
> bool postponed_def;
> } RelStatsInfo;
> comment /* 'r', 'v', 'c', etc */
> Is it wrong? there is no relkind='c'.
>
Yeah, I think 'c' should be an 'm' there.
> field postponed_def really deserves a comment.
>
Can do.
>
>
> we also need change
> enum dbObjectTypePriorities
> static const int dbObjectTypePriority[]
> ?
>
We'd need two entries, because that enum includes PRIO_PRE_DATA_BOUNDARY
and PRIO_POST_DATA_BOUNDARY, and statistics can either be in DATA or
POST_DATA.
>
>
> in dumpRelationStats, we can add Assert on it.
> if (!fout->dopt->dumpStatistics)
> return;
> Assert(dobj->dump & DUMP_COMPONENT_STATISTICS);
>
>
> I found out the owner's info is missing in the dumped content.
> for example, the line "Name: STATISTICS DATA tenk1_pkey;" missing owner
> info.
> not sure this is intended?
Good question. I'm not sure if we need it or not. If stats had an owner,
it'd be the owner of the relation.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-16 14:43 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-16 15:52 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-01-17 14:20 ` jian he <[email protected]>
2025-01-18 17:00 ` Re: Statistics Import and Export jian he <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: jian he @ 2025-01-17 14:20 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
hi.
SELECT * FROM pg_catalog.pg_restore_relation_stats(
'relation', 'public.tenk1_hundred'::regclass,
'version', '180000'::integer,
'relpages', '11'::integer,
'reltuples', '10000'::real,
'relallvisible', '0'::integer
);
dump and execute the above query generated a warning
WARNING: missing lock for relation "tenk1_hundred" (OID 18431,
relkind i) @ TID (15,34)
in dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
{
getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
dobj->name);
res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
appendAttStatsImport(out, fout, res);
PQclear(res)
}
i think, getAttStatsExportQuery no need to join catalog pg_namespace.
for example, after getAttStatsExportQuery, query->data looks like:
SELECT c.oid::regclass AS relation,
s.attname,s.inherited,
current_setting('server_version_num') AS version,
s.null_frac,
s.avg_width,
s.n_distinct,
s.most_common_vals,
s.most_common_freqs,
s.histogram_bounds,
s.correlation,
s.most_common_elems,
s.most_common_elem_freqs,
s.elem_count_histogram,
s.range_length_histogram,
s.range_empty_frac,
s.range_bounds_histogram
FROM pg_stats s JOIN pg_namespace n ON n.nspname = s.schemaname JOIN pg_class c
ON c.relname = s.tablename AND c.relnamespace = n.oid
WHERE s.schemaname = 'public'
AND s.tablename = 'varchar_tbl'
ORDER BY s.attname, s.inherited
-----------------------
The SELECT column list doesn't mention/use any of the pg_namespace columns.
```WHERE s.schemaname = 'public' AND s.tablename = 'varchar_tbl'``
" s.schemaname" combined with "s.tablename" will make sure that the
output is unique, at most one row.
i did a minor refactor about validation dumpData, dumpSchema,
dumpStatistics option.
I think it's more intuitive. first we process ``*only``option then
process no* option
Attachments:
[application/octet-stream] refactor_pg_dump_onlyoption.no-cfbot (2.0K, ../../CACJufxGreTY7qsCV8+Bkuv0p5SXGTScgh=D+Dq6=+_=XTp7FWg@mail.gmail.com/2-refactor_pg_dump_onlyoption.no-cfbot)
download
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-16 14:43 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-16 15:52 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-17 14:20 ` Re: Statistics Import and Export jian he <[email protected]>
@ 2025-01-18 17:00 ` jian he <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: jian he @ 2025-01-18 17:00 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Fri, Jan 17, 2025 at 10:20 PM jian he <[email protected]> wrote:
>
> hi.
>
> SELECT * FROM pg_catalog.pg_restore_relation_stats(
> 'relation', 'public.tenk1_hundred'::regclass,
> 'version', '180000'::integer,
> 'relpages', '11'::integer,
> 'reltuples', '10000'::real,
> 'relallvisible', '0'::integer
> );
> dump and execute the above query generated a warning
> WARNING: missing lock for relation "tenk1_hundred" (OID 18431,
> relkind i) @ TID (15,34)
>
This seems to be an existing issue.
For pg_restore_relation_stats, we don't have regress tests for index relation.
I am not sure the WARNING is ok.
I found out that the previous mail attached no-cfbot
(refactor_pg_dump_onlyoption.no-cfbot)
refactoring of statistics, data, schema is not fully correct.
This email attached no-cfbot,
I think it is tuitive and correct refactor of handling these three options.
typedef struct _dumpOptions, typedef struct _restoreOptions
we already have three bools (dumpSchema, dumpData, dumpStatistics).
Why do we need three int (no_data, no_schema, no_statistics) fields
for these two structs?
since they represent the same information. (for example, no_data == 1,
means/imply dumpData is false)
(disclaimer, this part I didn't dig deeper).
doc/src/sgml/ref/pg_restore.sgml
<varlistentry>
<term><option>-X</option></term>
<term><option>--statistics-only</option></term>
<listitem>
<para>
Restore only the statistics, not schema (data definitions) or data.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
uses the word <quote>schema</quote> in a different meaning.)
</para>
</listitem>
</varlistentry>
here, we don't need to mention
"(Do not confuse this with the <option>--schema</option> option, which"... part?
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'P'},
Here it should be
+ {"statistics-only", no_argument, NULL, 'X'},
?
If we introduced REQ_STATS, then better checking all the REQ_DATA occurrences,
does REQ_STATS apply to there also?
for example in pg_backup_tar.c and pg_backup_directory.c,
functions: WriteToc, function WriteDataChunks, RestoreArchive.
-----------------------------------------------
I tested locally, dump, restore, directory, custom format is not
working as intended, with v38.
I use the following to test it.
CONN2 is my local connect string.
BIN2 is a local bin directory.
varchar_tbl.dir is directory format dump full output, including data,
schema, statistics.
-----------------------------------------------
${CONN2} -c 'drop table varchar_tbl;'
$BIN2/pg_restore --dbname=src2 --list varchar_tbl.dir
#only schema
$BIN2/pg_restore --dbname=src2 --format=directory --no-statistics
--no-data varchar_tbl.dir
${CONN2} -c 'select attname=$$f1$$ as expect_zero_row from pg_stats
where tablename = $$varchar_tbl$$;'
${CONN2} -c 'select (reltuples < 0 and relpages = 0) as expect_true
from pg_class where relname = $$varchar_tbl$$;'
#only data
$BIN2/pg_restore --dbname=src2 --format=directory --no-statistics
--no-schema varchar_tbl.dir
${CONN2} -c 'select attname=$$f1$$ as expect_zero_row from pg_stats
where tablename = $$varchar_tbl$$;'
${CONN2} -c 'select (reltuples < 0 and relpages = 0) as expect_true
from pg_class where relname = $$varchar_tbl$$;'
#only statistics
$BIN2/pg_restore --dbname=src2 --format=directory --statistics-only
varchar_tbl.dir
${CONN2} -c 'select attname=$$f1$$ as expect_zero_row from pg_stats
where tablename = $$varchar_tbl$$;'
${CONN2} -c 'select reltuples > 0 and relpages > 0 as expect_true from
pg_class where relname = $$varchar_tbl$$;'
Attachments:
[application/octet-stream] v1-0001-misc-minor-refactoring.no-cfbot (2.7K, ../../CACJufxF_mytGCQKhycCHp2gwEse5WJvHWnJA-781c1BD8SsRkA@mail.gmail.com/2-v1-0001-misc-minor-refactoring.no-cfbot)
download
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-01-20 08:21 ` jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: jian he @ 2025-01-20 08:21 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Thu, Jan 16, 2025 at 4:56 AM Corey Huinker <[email protected]> wrote:
>>
>>
>>
>> I do like the idea of a "Statistics for ..." prefix, and I think it's
>> doable.
>
>
> And that's now implemented.
>
>
>> The caller needs some knowledge about that anyway, to correctly output
>> the statistics dump when the schema is not requested. Tests should
>> cover those cases, too.
>
>
> Tests for pg_dump --no-statistics and pg_dump --schema-only were added. Rebased to master as of today.
>
> I'm not completely happy with this patch, as I had to comment out one check in pg_backup_archiver that seemed necessary, but perhaps another set of eyes will set me straight.
>
> Attached is just the pg_dump stuff, and only for relation/attribute stats. The extended stats and vacuumdb work will be in their own threads going forward.
hi
The current v38 implementation allows statistics to be placed in
either SECTION_DATA or SECTION_POST_DATA.
IMHO, moving all statistics to the SECTION_POST_DATA section would
simplify things.
Attached is a patch that implements this change, (based on your patch,
obviously)
Reasoning making statistics in SECTION_POST_DATA are:
* statistics in multi sections will make the --section
handle statistics more harder for pg_dump and pg_restore.
* current doc in --schema-only says
"It is similar to, but for historical reasons not identical to, specifying
--section=pre-data --section=post-data.".
section span two sections making this sentence not less accurate.
Also, if we want to use --section option to dump all the statistics, we
need to use --data and --post-data together to get all the statistics.
* generally, --post-data section takes less time then --data section,
so putting it in SECTION_POST_DATA won't "cost" us that much.
* repairDependencyLoop, repairMatViewBoundaryMultiLoop is quite hard
to comprehend.
make statistics in SECTION_POST_DATA can make use not to think about
these changes.
also seems there is no materialized view statistics dump and restore
tests in src/bin/pg_dump/t/002_pg_dump.pl
* There are many REQ_DATA or REQ_SCHEMA occurrences, for each
occurrence, we may need to consider REQ_STATS.
make the statistics in SECTION_POST_DATA, then we don't need REQ_STATS.
what do you think?
This issue [1] seems not yet resolved.
[1] https://postgr.es/m/[email protected]
even if pg_upgrade has options --with-statistics and --with-statistics.
we still need a sentence to mention which option is default?
Attachments:
[application/octet-stream] v38-0001-make-statistics-dumped-at-SECTION_POST_DATA.no-cfbot (51.9K, ../../CACJufxGuRPfd70NwogkRE26cZcKGrR9gjG4OiH18tu=BnyezHA@mail.gmail.com/2-v38-0001-make-statistics-dumped-at-SECTION_POST_DATA.no-cfbot)
download
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
@ 2025-01-20 15:58 ` Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-01-20 15:58 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Mon, Jan 20, 2025 at 3:22 AM jian he <[email protected]> wrote:
> On Thu, Jan 16, 2025 at 4:56 AM Corey Huinker <[email protected]>
> wrote:
> >>
> >>
> >>
> >> I do like the idea of a "Statistics for ..." prefix, and I think it's
> >> doable.
> >
> >
> > And that's now implemented.
> >
> >
> >> The caller needs some knowledge about that anyway, to correctly output
> >> the statistics dump when the schema is not requested. Tests should
> >> cover those cases, too.
> >
> >
> > Tests for pg_dump --no-statistics and pg_dump --schema-only were added.
> Rebased to master as of today.
> >
> > I'm not completely happy with this patch, as I had to comment out one
> check in pg_backup_archiver that seemed necessary, but perhaps another set
> of eyes will set me straight.
> >
> > Attached is just the pg_dump stuff, and only for relation/attribute
> stats. The extended stats and vacuumdb work will be in their own threads
> going forward.
>
> hi
> The current v38 implementation allows statistics to be placed in
> either SECTION_DATA or SECTION_POST_DATA.
> IMHO, moving all statistics to the SECTION_POST_DATA section would
> simplify things.
> Attached is a patch that implements this change, (based on your patch,
> obviously)
>
That is where all statistics were previously. Others felt very strongly
that they should be mixed in to SECTION_DATA and SECTION_POST_DATA.
> * repairDependencyLoop, repairMatViewBoundaryMultiLoop is quite hard
to comprehend.
>
I don't disagree.
>
> * There are many REQ_DATA or REQ_SCHEMA occurrences, for each
> occurrence, we may need to consider REQ_STATS.
>
That is already in the works. Hoping to get that patch out soon.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-01-20 19:11 ` Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2025-01-20 19:11 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; jian he <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Mon, 2025-01-20 at 10:58 -0500, Corey Huinker wrote:
> That is where all statistics were previously. Others felt very
> strongly that they should be mixed in to SECTION_DATA and
> SECTION_POST_DATA.
I believe you are referring to Tom's statement that "it'll be a
serious, serious error for [stats] not to be SECTION_DATA". The
statement is somewhat softened by the sentence that follows, and
slightly more by [2]. But it's pretty clear that SECTION_POST_DATA is,
at best, an implementation comprosmise.
The reason we need to put some stats in SECTION_POST_DATA is because of
the hack to resolve MVs that depend on primary keys by moving the MV
into SECTION_POST_DATA. (An MV can depend on a primary key when the
query has a GROUP BY that relies on functional dependencies to be
valid.) That's a fairly marginal case, and one we might be able to
resolve a better way in the future, so I don't think that should drive
the design.
Reagrding [2] and [3], we might need to reconsider the behavior of the
--data-only option. I asked for the v38 behavior out of a sense of
consistency and completeness (the ability to express whatever
combination of things the user might want). But re-reading those
messages, we might want --data-only to include the stats?
Regards,
Jeff Davis
[1]
https://www.postgresql.org/message-id/[email protected]
[2]
https://www.postgresql.org/message-id/3156140.1713817153%40sss.pgh.pa.us
[3]
https://www.postgresql.org/message-id/3228677.1713844341%40sss.pgh.pa.us
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-01-20 21:45 ` Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-01-20 21:45 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
> I believe you are referring to Tom's statement that "it'll be a
> serious, serious error for [stats] not to be SECTION_DATA". The
> statement is somewhat softened by the sentence that follows, and
> slightly more by [2]. But it's pretty clear that SECTION_POST_DATA is,
> at best, an implementation comprosmise.
>
> The reason we need to put some stats in SECTION_POST_DATA is because of
> the hack to resolve MVs that depend on primary keys by moving the MV
> into SECTION_POST_DATA. (An MV can depend on a primary key when the
> query has a GROUP BY that relies on functional dependencies to be
> valid.) That's a fairly marginal case, and one we might be able to
> resolve a better way in the future, so I don't think that should drive
> the design.
>
I understand the benefits of having statistics on the underlying tables
could aid the performance of the queries that populate the materialized
views. What I struggle to understand is how that purpose isn't served
better by statistics being in SECTION_NONE like COMMENTs are, so that they
are imported immediately after the object that they reference.
>
> Reagrding [2] and [3], we might need to reconsider the behavior of the
> --data-only option. I asked for the v38 behavior out of a sense of
> consistency and completeness (the ability to express whatever
> combination of things the user might want). But re-reading those
> messages, we might want --data-only to include the stats?
I think there's going to be some friction in the user's shift from thinking
that they did want only data to realizing that they actually didn't want
schema.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-01-20 23:31 ` Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2025-01-20 23:31 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Mon, 2025-01-20 at 16:45 -0500, Corey Huinker wrote:
>
> What I struggle to understand is how that purpose isn't served better
> by statistics being in SECTION_NONE like COMMENTs are, so that they
> are imported immediately after the object that they reference.
Tom, you expressed the strongest opinions on this point, can you expand
a bit?
If I understand correctly:
* We strongly want stats to be exported by default[1].
* Adding a SECTION_STATS could work, but would be non-trivial and might
break expectations about the set of sections available[2].
* SECTION_NONE doesn't seem right. There would be no way to get the
stats using --section. Also, if there is no section boundary for the
stats, then couldn't they appear in a surprising order?
* I'm not sure about placing stats in SECTION_POST_DATA. That doesn't
seem terrible to me, but not great either.
* I'm also not 100% sure about the flags. The default should dump the
stats, of course. And I like the idea of allowing any combination of
schema, data and stats to be exported. But that leaves a wrinkle for --
data-only, which (as of v38) does not dump stats, because stats are a
third kind of thing. Perhaps stats should be expressed as a subtype of
data somehow, but I'm not sure exactly how.
Regards,
Jeff Davis
[1]
https://www.postgresql.org/message-id/3228677.1713844341%40sss.pgh.pa.us
[2]
https://www.postgresql.org/message-id/3156140.1713817153%40sss.pgh.pa.us
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-01-27 14:05 ` jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: jian he @ 2025-01-27 14:05 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Corey Huinker <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Tue, Jan 21, 2025 at 7:31 AM Jeff Davis <[email protected]> wrote:
>
> On Mon, 2025-01-20 at 16:45 -0500, Corey Huinker wrote:
> >
> > What I struggle to understand is how that purpose isn't served better
> > by statistics being in SECTION_NONE like COMMENTs are, so that they
> > are imported immediately after the object that they reference.
>
> Tom, you expressed the strongest opinions on this point, can you expand
> a bit?
>
> If I understand correctly:
>
> * We strongly want stats to be exported by default[1].
>
> * Adding a SECTION_STATS could work, but would be non-trivial and might
> break expectations about the set of sections available[2].
>
> * SECTION_NONE doesn't seem right. There would be no way to get the
> stats using --section. Also, if there is no section boundary for the
> stats, then couldn't they appear in a surprising order?
>
> * I'm not sure about placing stats in SECTION_POST_DATA. That doesn't
> seem terrible to me, but not great either.
>
index is on SECTION_POST_DATA.
To dump all the statistics, we have to go through SECTION_POST_DATA.
place it there would be more convenient.
Tomas Vondra also mentioned this on [1]
[1] https://www.postgresql.org/message-id/bf724b21-914a-4497-84e3-49944f9776f6%40enterprisedb.com
> * I'm also not 100% sure about the flags. The default should dump the
> stats, of course. And I like the idea of allowing any combination of
> schema, data and stats to be exported. But that leaves a wrinkle for --
> data-only, which (as of v38) does not dump stats, because stats are a
> third kind of thing. Perhaps stats should be expressed as a subtype of
> data somehow, but I'm not sure exactly how.
>
if we have --data-only, --schema-only, --statistics-only, three options, then
--data-only also dump statistics would be unintuitive?
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
@ 2025-01-27 16:09 ` Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-01-27 16:09 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Mon, Jan 27, 2025 at 9:05 AM jian he <[email protected]> wrote:
> On Tue, Jan 21, 2025 at 7:31 AM Jeff Davis <[email protected]> wrote:
> >
> > On Mon, 2025-01-20 at 16:45 -0500, Corey Huinker wrote:
> > >
> > > What I struggle to understand is how that purpose isn't served better
> > > by statistics being in SECTION_NONE like COMMENTs are, so that they
> > > are imported immediately after the object that they reference.
> >
> > Tom, you expressed the strongest opinions on this point, can you expand
> > a bit?
> >
> > If I understand correctly:
> >
> > * We strongly want stats to be exported by default[1].
> >
> > * Adding a SECTION_STATS could work, but would be non-trivial and might
> > break expectations about the set of sections available[2].
> >
> > * SECTION_NONE doesn't seem right. There would be no way to get the
> > stats using --section. Also, if there is no section boundary for the
> > stats, then couldn't they appear in a surprising order?
> >
> > * I'm not sure about placing stats in SECTION_POST_DATA. That doesn't
> > seem terrible to me, but not great either.
> >
>
> index is on SECTION_POST_DATA.
> To dump all the statistics, we have to go through SECTION_POST_DATA.
> place it there would be more convenient.
>
That would be the simpler solution, but those statistics may come in handy
for refreshing mviews, so some may want table stats to stay in SECTION_DATA.
>
> Tomas Vondra also mentioned this on [1]
> [1]
> https://www.postgresql.org/message-id/bf724b21-914a-4497-84e3-49944f9776f6%40enterprisedb.com
>
> > * I'm also not 100% sure about the flags. The default should dump the
> > stats, of course. And I like the idea of allowing any combination of
> > schema, data and stats to be exported. But that leaves a wrinkle for --
> > data-only, which (as of v38) does not dump stats, because stats are a
> > third kind of thing. Perhaps stats should be expressed as a subtype of
> > data somehow, but I'm not sure exactly how.
> >
> if we have --data-only, --schema-only, --statistics-only, three options,
> then
> --data-only also dump statistics would be unintuitive?
>
Yeah, I think the codebase and the user flags both have confusing bits
where the not-wanting of one type of thing was specified by only-wanting
the other thing, and those choices fall apart when the binary becomes
trinary.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-06 04:01 ` Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-02-06 04:01 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Jeff Davis <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Mon, Jan 27, 2025 at 11:09 AM Corey Huinker <[email protected]>
wrote:
> On Mon, Jan 27, 2025 at 9:05 AM jian he <[email protected]>
> wrote:
>
>> On Tue, Jan 21, 2025 at 7:31 AM Jeff Davis <[email protected]> wrote:
>> >
>> > On Mon, 2025-01-20 at 16:45 -0500, Corey Huinker wrote:
>> > >
>> > > What I struggle to understand is how that purpose isn't served better
>> > > by statistics being in SECTION_NONE like COMMENTs are, so that they
>> > > are imported immediately after the object that they reference.
>> >
>> > Tom, you expressed the strongest opinions on this point, can you expand
>> > a bit?
>> >
>> > If I understand correctly:
>> >
>> > * We strongly want stats to be exported by default[1].
>> >
>> > * Adding a SECTION_STATS could work, but would be non-trivial and might
>> > break expectations about the set of sections available[2].
>> >
>> > * SECTION_NONE doesn't seem right. There would be no way to get the
>> > stats using --section. Also, if there is no section boundary for the
>> > stats, then couldn't they appear in a surprising order?
>> >
>> > * I'm not sure about placing stats in SECTION_POST_DATA. That doesn't
>> > seem terrible to me, but not great either.
>> >
>>
>> index is on SECTION_POST_DATA.
>> To dump all the statistics, we have to go through SECTION_POST_DATA.
>> place it there would be more convenient.
>>
>
> That would be the simpler solution, but those statistics may come in handy
> for refreshing mviews, so some may want table stats to stay in SECTION_DATA.
>
>
>>
>> Tomas Vondra also mentioned this on [1]
>> [1]
>> https://www.postgresql.org/message-id/bf724b21-914a-4497-84e3-49944f9776f6%40enterprisedb.com
>>
>> > * I'm also not 100% sure about the flags. The default should dump the
>> > stats, of course. And I like the idea of allowing any combination of
>> > schema, data and stats to be exported. But that leaves a wrinkle for --
>> > data-only, which (as of v38) does not dump stats, because stats are a
>> > third kind of thing. Perhaps stats should be expressed as a subtype of
>> > data somehow, but I'm not sure exactly how.
>> >
>> if we have --data-only, --schema-only, --statistics-only, three options,
>> then
>> --data-only also dump statistics would be unintuitive?
>>
>
> Yeah, I think the codebase and the user flags both have confusing bits
> where the not-wanting of one type of thing was specified by only-wanting
> the other thing, and those choices fall apart when the binary becomes
> trinary.
>
Seems I also replied only to Micahel with the v45 patch.
And here's an update to the pg_dump code itself. This currently has failing
TAP tests for statistics in the custom and dir formats, but is working
otherwise.
Attachments:
[text/x-patch] v45-0001-Lock-table-first-when-setting-index-relation-sta.patch (8.2K, ../../CADkLM=cnYxaNFbAQRMLeOOga0j5pY_V6Tfcn+vjm_rXL=WOF7Q@mail.gmail.com/3-v45-0001-Lock-table-first-when-setting-index-relation-sta.patch)
download | inline diff:
From a2948c851191977af6fe22f79a13589204a20df8 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Tue, 21 Jan 2025 11:52:58 -0500
Subject: [PATCH v45 1/2] Lock table first when setting index relation
statistics.
Jian He reported [1] a missing lock relation bug in
pg_restore_relation_stats when restoring stats to an index.
This fix follows the steps for proper locking prior to an inplace update
of a relation as specified in aac2c9b4fde8, and mimics the locking
behavior of the analyze command, as well as correctly doing the ACL
checks against the underlying relation of an index rather than the index
itself.
There is no special case for partitioned indexes, so while we want to
the ACL checks against the underlying relation, we need to take out
the more restrictive ShareUpdateExclusiveLock on the partitioned index.
[1] https://www.postgresql.org/message-id/CACJufxGreTY7qsCV8%2BBkuv0p5SXGTScgh%3DD%2BDq6%3D%2B_%3DXTp7FWg%40mail.gmail.com
---
src/backend/statistics/stat_utils.c | 45 +++++++++++++++---
src/test/regress/expected/stats_import.out | 53 ++++++++++++++++++++++
src/test/regress/sql/stats_import.sql | 36 +++++++++++++++
3 files changed, 128 insertions(+), 6 deletions(-)
diff --git a/src/backend/statistics/stat_utils.c b/src/backend/statistics/stat_utils.c
index 0d446f55b0..f87007e72c 100644
--- a/src/backend/statistics/stat_utils.c
+++ b/src/backend/statistics/stat_utils.c
@@ -17,13 +17,16 @@
#include "postgres.h"
#include "access/relation.h"
+#include "catalog/index.h"
#include "catalog/pg_database.h"
#include "funcapi.h"
#include "miscadmin.h"
#include "statistics/stat_utils.h"
+#include "storage/lmgr.h"
#include "utils/acl.h"
#include "utils/array.h"
#include "utils/builtins.h"
+#include "utils/lsyscache.h"
#include "utils/rel.h"
/*
@@ -126,18 +129,45 @@ stats_check_arg_pair(FunctionCallInfo fcinfo,
void
stats_lock_check_privileges(Oid reloid)
{
- Relation rel = relation_open(reloid, ShareUpdateExclusiveLock);
- const char relkind = rel->rd_rel->relkind;
+ Relation rel;
+ Oid relation_oid = reloid;
+ Oid index_oid = InvalidOid;
+ LOCKMODE index_lockmode = AccessShareLock;
- /* All of the types that can be used with ANALYZE, plus indexes */
- switch (relkind)
+ /*
+ * For indexes, we follow what do_analyze_rel() does so as to avoid any
+ * deadlocks with analyze/vacuum, which is to take out a
+ * ShareUpdateExclusive on table/matview first and then take an
+ * AccessShareLock on the index itself. See check_inplace_rel_lock()
+ * to see how this special case is implemented.
+ *
+ * Partitioned indexes do not have an exception in check_inplace_rel_lock(),
+ * so we want to take a ShareUpdateExclusive lock there instead.
+ */
+ switch(get_rel_relkind(reloid))
{
- case RELKIND_RELATION:
case RELKIND_INDEX:
+ relation_oid = IndexGetRelation(reloid, false);
+ index_oid = reloid;
+ break;
+ case RELKIND_PARTITIONED_INDEX:
+ relation_oid = IndexGetRelation(reloid, false);
+ index_oid = reloid;
+ index_lockmode = ShareUpdateExclusiveLock;
+ break;
+ default:
+ break;
+ }
+
+ rel = relation_open(relation_oid, ShareUpdateExclusiveLock);
+
+ switch (rel->rd_rel->relkind)
+ {
+ /* All of the types that can be used with ANALYZE */
+ case RELKIND_RELATION:
case RELKIND_MATVIEW:
case RELKIND_FOREIGN_TABLE:
case RELKIND_PARTITIONED_TABLE:
- case RELKIND_PARTITIONED_INDEX:
break;
default:
ereport(ERROR,
@@ -164,6 +194,9 @@ stats_lock_check_privileges(Oid reloid)
NameStr(rel->rd_rel->relname));
}
+ if (OidIsValid(index_oid))
+ LockRelationOid(index_oid, index_lockmode);
+
relation_close(rel, NoLock);
}
diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out
index fb50da1cd8..6cd584da68 100644
--- a/src/test/regress/expected/stats_import.out
+++ b/src/test/regress/expected/stats_import.out
@@ -85,6 +85,26 @@ WHERE oid = 'stats_import.test'::regclass;
17 | 400 | 4
(1 row)
+CREATE INDEX test_i ON stats_import.test(id);
+-- regular indexes have special case locking rules
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test_i'::regclass,
+ relpages => 18::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.test_i'::regclass,
+ 'relpages', 19::integer );
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- positional arguments
SELECT
pg_catalog.pg_set_relation_stats(
@@ -182,6 +202,7 @@ CREATE TABLE stats_import.part_child_1
PARTITION OF stats_import.part_parent
FOR VALUES FROM (0) TO (10)
WITH (autovacuum_enabled = false);
+CREATE INDEX part_parent_i ON stats_import.part_parent(i);
ANALYZE stats_import.part_parent;
SELECT relpages
FROM pg_class
@@ -202,6 +223,25 @@ SELECT
(1 row)
+-- Partitioned indexes aren't analyzed but it is possible to set stats.
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.part_parent_i'::regclass,
+ 'relpages', 2::integer);
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- nothing stops us from setting it to -1
SELECT
pg_catalog.pg_set_relation_stats(
@@ -1414,6 +1454,19 @@ SELECT 3, 'tre', (3, 3.3, 'TRE', '2003-03-03', NULL)::stats_import.complex_type,
UNION ALL
SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+-- Test for proper locking
+SELECT * FROM pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.is_odd'::regclass,
+ 'version', '180000'::integer,
+ 'relpages', '11'::integer,
+ 'reltuples', '10000'::real,
+ 'relallvisible', '0'::integer
+);
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- Generate statistics on table with data
ANALYZE stats_import.test;
CREATE TABLE stats_import.test_clone ( LIKE stats_import.test )
diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql
index d3058bf8f6..23e85fc6ba 100644
--- a/src/test/regress/sql/stats_import.sql
+++ b/src/test/regress/sql/stats_import.sql
@@ -64,6 +64,19 @@ SELECT relpages, reltuples, relallvisible
FROM pg_class
WHERE oid = 'stats_import.test'::regclass;
+CREATE INDEX test_i ON stats_import.test(id);
+
+-- regular indexes have special case locking rules
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test_i'::regclass,
+ relpages => 18::integer);
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.test_i'::regclass,
+ 'relpages', 19::integer );
+
-- positional arguments
SELECT
pg_catalog.pg_set_relation_stats(
@@ -127,6 +140,8 @@ CREATE TABLE stats_import.part_child_1
FOR VALUES FROM (0) TO (10)
WITH (autovacuum_enabled = false);
+CREATE INDEX part_parent_i ON stats_import.part_parent(i);
+
ANALYZE stats_import.part_parent;
SELECT relpages
@@ -140,6 +155,17 @@ SELECT
relation => 'stats_import.part_parent'::regclass,
relpages => 2::integer);
+-- Partitioned indexes aren't analyzed but it is possible to set stats.
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.part_parent_i'::regclass,
+ 'relpages', 2::integer);
+
-- nothing stops us from setting it to -1
SELECT
pg_catalog.pg_set_relation_stats(
@@ -1062,6 +1088,16 @@ SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+
+-- Test for proper locking
+SELECT * FROM pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.is_odd'::regclass,
+ 'version', '180000'::integer,
+ 'relpages', '11'::integer,
+ 'reltuples', '10000'::real,
+ 'relallvisible', '0'::integer
+);
+
-- Generate statistics on table with data
ANALYZE stats_import.test;
--
2.48.1
[text/x-patch] v45-0002-Enable-dumping-of-table-index-stats-in-pg_dump.patch (66.6K, ../../CADkLM=cnYxaNFbAQRMLeOOga0j5pY_V6Tfcn+vjm_rXL=WOF7Q@mail.gmail.com/4-v45-0002-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From 8b25dfd095dba60c78434ec812e06f3aad71e2f1 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v45 2/2] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute. These
statements will restore the statistics of the current system onto the
destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, statistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index and Materialized View
statistics are dumped in the post-data section.
Add --no-data option.
This option is useful for situations where someone wishes to test
query plans from a production database without copying production data.
This also makes the corresponding change to the simulated pg_upgrade in
the TAP tests for pg_dump.
This checks that dumping statistics is now the default, and that
--no-statistics will suppress statistics.
Add --no-schema option to pg_dump, etc.
Previously, users could use --data-only when they wanted to suppress
schema from a dump. However, that no longer makes sense now that the
data/schema binary has become the data/schema/statistics trinary.
---
src/bin/pg_dump/pg_backup.h | 10 +-
src/bin/pg_dump/pg_backup_archiver.c | 91 ++++--
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_backup_directory.c | 2 +-
src/bin/pg_dump/pg_dump.c | 395 +++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 11 +
src/bin/pg_dump/pg_dump_sort.c | 36 ++-
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 32 ++-
src/bin/pg_dump/t/001_basic.pl | 18 ++
src/bin/pg_dump/t/002_pg_dump.pl | 109 ++++++-
src/bin/pg_upgrade/dump.c | 6 +-
src/bin/pg_upgrade/option.c | 12 +
src/bin/pg_upgrade/pg_upgrade.h | 1 +
doc/src/sgml/ref/pg_dump.sgml | 69 ++++-
doc/src/sgml/ref/pg_dumpall.sgml | 38 +++
doc/src/sgml/ref/pg_restore.sgml | 51 +++-
doc/src/sgml/ref/pgupgrade.sgml | 18 ++
src/tools/pgindent/typedefs.list | 1 +
19 files changed, 847 insertions(+), 61 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b2..3fa1474fad 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,9 +110,12 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +163,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -179,8 +183,11 @@ typedef struct _dumpOptions
int column_inserts;
int if_exists;
int no_comments;
- int no_security_labels;
+ int no_data;
int no_publications;
+ int no_schema;
+ int no_security_labels;
+ int no_statistics;
int no_subscriptions;
int no_toast_compression;
int no_unlogged_table_data;
@@ -208,6 +215,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844..1cbe07c64f 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -46,6 +46,11 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
+typedef enum entryType {
+ default_entry,
+ data_entry,
+ statistics_entry
+} entryType;
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -53,7 +58,7 @@ static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr,
DataDirSyncMethod sync_method);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
-static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
+static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type);
static char *sanitize_line(const char *str, bool want_hyphen);
static void _doSetFixedOutputState(ArchiveHandle *AH);
static void _doSetSessionAuth(ArchiveHandle *AH, const char *user);
@@ -149,6 +154,7 @@ InitDumpOptions(DumpOptions *opts)
opts->dumpSections = DUMP_UNSECTIONED;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
}
/*
@@ -169,9 +175,10 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->outputClean = ropt->dropSchema;
dopt->dumpData = ropt->dumpData;
dopt->dumpSchema = ropt->dumpSchema;
+ dopt->dumpSections = ropt->dumpSections;
+ dopt->dumpStatistics = ropt->dumpStatistics;
dopt->if_exists = ropt->if_exists;
dopt->column_inserts = ropt->column_inserts;
- dopt->dumpSections = ropt->dumpSections;
dopt->aclsSkip = ropt->aclsSkip;
dopt->outputSuperuser = ropt->superuser;
dopt->outputCreateDB = ropt->createDB;
@@ -186,6 +193,9 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
@@ -418,30 +428,31 @@ RestoreArchive(Archive *AHX)
}
/*
- * Work out if we have an implied data-only restore. This can happen if
- * the dump was data only or if the user has used a toc list to exclude
- * all of the schema data. All we do is look for schema entries - if none
- * are found then we unset the dumpSchema flag.
+ * Work out if we have an schema-less restore. This can happen if the dump
+ * was data-only or statistics-only or no-schema or if the user has used a
+ * toc list to exclude all of the schema data. All we do is look for schema
+ * entries - if none are found then we unset the dumpSchema flag.
*
* We could scan for wanted TABLE entries, but that is not the same as
* data-only. At this stage, it seems unnecessary (6-Mar-2001).
*/
if (ropt->dumpSchema)
{
- int impliedDataOnly = 1;
+ bool no_schema_found = true;
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
+ /* TODO: should this now be (REQ_SCHEMA | REQ_STATS)? */
if ((te->reqs & REQ_SCHEMA) != 0)
- { /* It's schema, and it's wanted */
- impliedDataOnly = 0;
+ {
+ no_schema_found = false;
break;
}
}
- if (impliedDataOnly)
+ if (no_schema_found)
{
ropt->dumpSchema = false;
- pg_log_info("implied data-only restore");
+ pg_log_info("implied no-schema restore");
}
}
@@ -739,7 +750,7 @@ RestoreArchive(Archive *AHX)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
continue; /* ignore if not to be dumped at all */
switch (_tocEntryRestorePass(te))
@@ -760,7 +771,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -770,7 +781,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_POST_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -869,7 +880,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
pg_log_info("creating %s \"%s\"",
te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
defnDumped = true;
if (strcmp(te->desc, "TABLE") == 0)
@@ -938,7 +949,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
*/
if (AH->PrintTocDataPtr != NULL)
{
- _printTocEntry(AH, te, true);
+ _printTocEntry(AH, te, data_entry);
if (strcmp(te->desc, "BLOBS") == 0 ||
strcmp(te->desc, "BLOB COMMENTS") == 0)
@@ -1036,15 +1047,21 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
{
/* If we haven't already dumped the defn part, do so now */
pg_log_info("executing %s %s", te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
}
}
+ /*
+ * If it has a statistics component that we want, then process that
+ */
+ if ((reqs & REQ_STATS) != 0)
+ _printTocEntry(AH, te, statistics_entry);
+
/*
* If we emitted anything for this TOC entry, that counts as one action
* against the transaction-size limit. Commit if it's time to.
*/
- if ((reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && ropt->txn_size > 0)
+ if ((reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && ropt->txn_size > 0)
{
if (++AH->txnCount >= ropt->txn_size)
{
@@ -1084,6 +1101,7 @@ NewRestoreOptions(void)
opts->compression_spec.level = 0;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
return opts;
}
@@ -1329,7 +1347,7 @@ PrintTOCSummary(Archive *AHX)
te->reqs = _tocEntryRequired(te, curSection, AH);
/* Now, should we print it? */
if (ropt->verbose ||
- (te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0)
+ (te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0)
{
char *sanitized_name;
char *sanitized_schema;
@@ -2918,6 +2936,14 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strcmp(te->desc, "SEARCHPATH") == 0)
return REQ_SPECIAL;
+ if (strcmp(te->desc, "STATISTICS DATA") == 0)
+ {
+ if (!ropt->dumpStatistics)
+ return 0;
+ else
+ res = REQ_STATS; /* return REQ_STATS; */
+ }
+
/*
* DATABASE and DATABASE PROPERTIES also have a special rule: they are
* restored in createDB mode, and not restored otherwise, independently of
@@ -2962,6 +2988,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's statistics and we don't want statistics, maybe ignore it */
+ if (!ropt->dumpStatistics && strcmp(te->desc, "STATISTICS DATA") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +3021,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS DATA") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
@@ -3107,6 +3138,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
}
+
/*
* Determine whether the TOC entry contains schema and/or data components,
* and mask off inapplicable REQ bits. If it had a dataDumper, assume
@@ -3172,12 +3204,12 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strncmp(te->tag, "LARGE OBJECT", 12) == 0) ||
(strcmp(te->desc, "SECURITY LABEL") == 0 &&
strncmp(te->tag, "LARGE OBJECT", 12) == 0))))
- res = res & REQ_SCHEMA;
+ res = res & (REQ_SCHEMA | REQ_STATS);
}
/* Mask it if we don't want schema */
if (!ropt->dumpSchema)
- res = res & REQ_DATA;
+ res = res & (REQ_DATA | REQ_STATS);
return res;
}
@@ -3729,7 +3761,7 @@ _getObjectDescription(PQExpBuffer buf, const TocEntry *te)
* will remain at default, until the matching ACL TOC entry is restored.
*/
static void
-_printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
+_printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type)
{
RestoreOptions *ropt = AH->public.ropt;
@@ -3753,10 +3785,17 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
char *sanitized_schema;
char *sanitized_owner;
- if (isData)
- pfx = "Data for ";
- else
- pfx = "";
+ switch (entry_type)
+ {
+ case data_entry:
+ pfx = "Data for ";
+ break;
+ case statistics_entry:
+ pfx = "Statistics for ";
+ break;
+ default:
+ pfx = "";
+ }
ahprintf(AH, "--\n");
if (AH->public.verbose)
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index ce5ed1dd39..a2064f471e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -209,7 +209,8 @@ typedef enum
#define REQ_SCHEMA 0x01 /* want schema */
#define REQ_DATA 0x02 /* want data */
-#define REQ_SPECIAL 0x04 /* for special TOC entries */
+#define REQ_STATS 0x04
+#define REQ_SPECIAL 0x08 /* for special TOC entries */
struct _archiveHandle
{
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 240a1d4106..b2a841bb0f 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -780,7 +780,7 @@ _PrepParallelRestore(ArchiveHandle *AH)
continue;
/* We may ignore items not due to be restored */
- if ((te->reqs & REQ_DATA) == 0)
+ if ((te->reqs & (REQ_DATA | REQ_STATS)) == 0)
continue;
/*
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index af857f00c7..60621302dd 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -431,6 +431,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -467,6 +468,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -493,8 +495,11 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -540,7 +545,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -614,6 +619,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -785,6 +794,17 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -799,8 +819,9 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
@@ -1100,6 +1121,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1178,7 +1200,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1191,11 +1213,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1220,8 +1243,11 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
+ printf(_(" --no-schema do not dump schema\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6778,6 +6804,43 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+ info->postponed_def = false;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7155,6 +7218,9 @@ getTables(Archive *fout, int *numTables)
/* Tables have data */
tblinfo[i].dobj.components |= DUMP_COMPONENT_DATA;
+ /*
+ tblinfo[i].dobj.components |= DUMP_COMPONENT_STATISTICS;
+ */
/* Mark whether table has an ACL */
if (!PQgetisnull(res, i, i_relacl))
@@ -7203,6 +7269,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting || dopt->dumpStatistics)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7649,11 +7717,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7676,7 +7747,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7710,6 +7788,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10297,6 +10377,296 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * Decide which section to use based on the relkind of the parent object.
+ *
+ * NB: materialized views may be postponed from SECTION_PRE_DATA to
+ * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the
+ * matview stats will also be postponed to SECTION_POST_DATA. See
+ * repairMatViewBoundaryMultiLoop().
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+ switch (rsinfo->relkind)
+ {
+ case RELKIND_RELATION:
+ case RELKIND_PARTITIONED_TABLE:
+ case RELKIND_MATVIEW:
+ return SECTION_DATA;
+ case RELKIND_INDEX:
+ case RELKIND_PARTITIONED_INDEX:
+ return SECTION_POST_DATA;
+ default:
+ pg_fatal("cannot dump statistics for relation kind '%c'",
+ rsinfo->relkind);
+ }
+
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+ DumpId *deps = NULL;
+ int ndeps = 0;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ /* dependent on the relation definition, if doing schema */
+ if (fout->dopt->dumpSchema)
+ {
+ deps = dobj->dependencies;
+ ndeps = dobj->nDeps;
+ }
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = rsinfo->postponed_def ?
+ SECTION_POST_DATA : statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = deps,
+ .nDeps = ndeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10745,6 +11115,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -17184,6 +17557,8 @@ dumpIndex(Archive *fout, const IndxInfo *indxinfo)
free(indstatvalsarray);
}
+ /* Comments and stats share same .dep */
+
/* Dump Index Comments */
if (indxinfo->dobj.dump & DUMP_COMPONENT_COMMENT)
dumpComment(fout, "INDEX", qindxname,
@@ -18971,6 +19346,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 7139c88a69..410c162a85 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -83,10 +83,13 @@ typedef enum
DO_PUBLICATION,
DO_PUBLICATION_REL,
DO_PUBLICATION_TABLE_IN_SCHEMA,
+ DO_REL_STATS,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
} DumpableObjectType;
+#define NUM_DUMPABLE_OBJECT_TYPES (DO_SUBSCRIPTION_REL + 1)
+
/*
* DumpComponents is a bitmask of the potentially dumpable components of
* a database object: its core definition, plus optional attributes such
@@ -110,6 +113,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -430,6 +434,13 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'm', 'i', etc */
+ bool postponed_def; /* stats must be postponed into post-data */
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index dc9a28924b..201eeedde7 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -81,6 +81,7 @@ enum dbObjectTypePriorities
PRIO_TABLE_DATA,
PRIO_SEQUENCE_SET,
PRIO_LARGE_OBJECT_DATA,
+ PRIO_STATISTICS_DATA_DATA,
PRIO_POST_DATA_BOUNDARY, /* boundary! */
PRIO_CONSTRAINT,
PRIO_INDEX,
@@ -148,11 +149,12 @@ static const int dbObjectTypePriority[] =
[DO_PUBLICATION] = PRIO_PUBLICATION,
[DO_PUBLICATION_REL] = PRIO_PUBLICATION_REL,
[DO_PUBLICATION_TABLE_IN_SCHEMA] = PRIO_PUBLICATION_TABLE_IN_SCHEMA,
+ [DO_REL_STATS] = PRIO_STATISTICS_DATA_DATA,
[DO_SUBSCRIPTION] = PRIO_SUBSCRIPTION,
[DO_SUBSCRIPTION_REL] = PRIO_SUBSCRIPTION_REL,
};
-StaticAssertDecl(lengthof(dbObjectTypePriority) == (DO_SUBSCRIPTION_REL + 1),
+StaticAssertDecl(lengthof(dbObjectTypePriority) == NUM_DUMPABLE_OBJECT_TYPES,
"array length mismatch");
static DumpId preDataBoundId;
@@ -801,11 +803,21 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj,
{
/* remove boundary's dependency on object after it in loop */
removeObjectDependency(boundaryobj, nextobj->dumpId);
- /* if that object is a matview, mark it as postponed into post-data */
+ /*
+ * If that object is a matview or matview status, mark it as postponed into
+ * post-data.
+ */
if (nextobj->objType == DO_TABLE)
{
TableInfo *nextinfo = (TableInfo *) nextobj;
+ if (nextinfo->relkind == RELKIND_MATVIEW)
+ nextinfo->postponed_def = true;
+ }
+ else if (nextobj->objType == DO_REL_STATS)
+ {
+ RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj;
+
if (nextinfo->relkind == RELKIND_MATVIEW)
nextinfo->postponed_def = true;
}
@@ -1018,6 +1030,21 @@ repairDependencyLoop(DumpableObject **loop,
{
DumpableObject *nextobj;
+ nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
+ repairMatViewBoundaryMultiLoop(loop[j], nextobj);
+ return;
+ }
+ }
+ }
+ else if (loop[i]->objType == DO_REL_STATS &&
+ ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW)
+ {
+ for (j = 0; j < nLoop; j++)
+ {
+ if (loop[j]->objType == DO_POST_DATA_BOUNDARY)
+ {
+ DumpableObject *nextobj;
+
nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
repairMatViewBoundaryMultiLoop(loop[j], nextobj);
return;
@@ -1500,6 +1527,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 396f79781c..7effb70490 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index c602272d7d..fa08f0a186 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,9 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool data_only = false;
+ bool schema_only = false;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -71,12 +74,13 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_comments = 0;
+ static int no_data = 0;
static int no_publications = 0;
+ static int no_schema = 0;
static int no_security_labels = 0;
+ static int no_statistics = 0;
static int no_subscriptions = 0;
static int strict_names = 0;
- bool data_only = false;
- bool schema_only = false;
struct option cmdopts[] = {
{"clean", 0, NULL, 'c'},
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -124,9 +129,12 @@ main(int argc, char **argv)
{"transaction-size", required_argument, NULL, 5},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
@@ -271,6 +279,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +355,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +378,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpData = data_only || (!no_data && !schema_only && !statistics_only);
+ opts->dumpSchema = schema_only || (!no_schema && !data_only && !statistics_only);
+ opts->dumpStatistics = statistics_only || (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +392,8 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
@@ -482,6 +501,7 @@ usage(const char *progname)
printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n"));
printf(_(" -T, --trigger=NAME restore named trigger\n"));
printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only restore only the statistics, not schema or data\n"));
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
@@ -489,10 +509,14 @@ usage(const char *progname)
" in FILENAME\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comment commands\n"));
+ printf(_(" --no-data do not restore data\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
+ /* This hack is only needed in a data-only restore */
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 214240f1ae..f29da06ed2 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 805ba9f49f..7e1f392d21 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -66,6 +66,7 @@ my %pgdump_runs = (
'--file' => "$tempdir/binary_upgrade.dump",
'--no-password',
'--schema-only',
+ '--no-data',
'--binary-upgrade',
'--dbname' => 'postgres', # alternative way to specify database
],
@@ -710,6 +711,39 @@ my %pgdump_runs = (
'--no-large-objects',
'postgres',
],
+ },
+ no_statistics => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_statistics.sql",
+ '--no-statistics',
+ 'postgres',
+ ],
+ },
+ no_data_no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_data_no_schema.sql",
+ '--no-data',
+ '--no-schema',
+ 'postgres',
+ ],
+ },
+ statistics_only => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/statistics_only.sql",
+ '--statistics-only',
+ 'postgres',
+ ],
+ },
+ no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_schema.sql",
+ '--no-schema',
+ 'postgres',
+ ],
},);
###############################################################
@@ -776,6 +810,7 @@ my %full_runs = (
no_large_objects => 1,
no_owner => 1,
no_privs => 1,
+ no_statistics => 1,
no_table_access_method => 1,
pg_dumpall_dbprivs => 1,
pg_dumpall_exclude => 1,
@@ -977,6 +1012,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1390,6 +1426,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1411,6 +1448,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1432,6 +1470,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1598,6 +1637,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1751,6 +1791,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_test_table => 1,
section_data => 1,
},
@@ -1778,6 +1819,7 @@ my %tests = (
data_only => 1,
exclude_test_table => 1,
exclude_test_table_data => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1798,7 +1840,10 @@ my %tests = (
\QCOPY dump_test.fk_reference_test_table (col1) FROM stdin;\E
\n(?:\d\n){5}\\\.\n
/xms,
- like => { data_only => 1, },
+ like => {
+ data_only => 1,
+ no_schema => 1,
+ },
},
'COPY test_second_table' => {
@@ -1814,6 +1859,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1836,6 +1882,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1859,6 +1906,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1881,6 +1929,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1903,6 +1952,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -3299,6 +3349,7 @@ my %tests = (
like => {
%full_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
only_dump_test_schema => 1,
test_schema_plus_large_objects => 1,
@@ -3469,6 +3520,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_measurement => 1,
section_data => 1,
only_dump_test_schema => 1,
@@ -4351,6 +4403,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
binary_upgrade => 1,
@@ -4651,6 +4704,60 @@ my %tests = (
},
},
+ #
+ # Table statistics should go in section=data.
+ # Materialized view statistics should go in section=post-data.
+ #
+ # TABLE and MATVIEW stats will end up in SECTION_DATA.
+ # INDEX stats (expression columns only) will end up in SECTION_POST_DATA.
+ #
+ 'statistics_import' => {
+ create_sql => '
+ CREATE TABLE dump_test.has_stats
+ AS SELECT g.g AS x, g.g / 2 AS y FROM generate_series(1,100) AS g(g);
+ CREATE MATERIALIZED VIEW dump_test.has_stats_mv AS SELECT * FROM dump_test.has_stats;
+ CREATE INDEX dup_test_post_data_ix ON dump_test.has_stats((x - 1));
+ ANALYZE dump_test.has_stats, dump_test.has_stats_mv;',
+ regexp => qr/pg_catalog.pg_restore_attribute_stats/,
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ exclude_dump_test_schema => 1,
+ no_statistics => 1,
+ only_dump_measurement => 1,
+ schema_only => 1,
+ },
+ },
+
+ 'relstats_on_unanalyzed_tables' => {
+ regexp => qr/pg_catalog.pg_restore_relation_stats/,
+
+ # this shouldn't ever get emitted anymore
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ only_dump_test_table => 1,
+ role => 1,
+ role_parallel => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ no_statistics => 1,
+ schema_only => 1,
+ },
+ },
+
# CREATE TABLE with partitioned table and various AMs. One
# partition uses the same default as the parent, and a second
# uses its own AM.
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8ce0fa3020..a29cd2cca9 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 108eb7a1ba..3b6c7ec994 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -306,7 +316,9 @@ usage(void)
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
+ printf(_(" --no-statistics do not import statistics from old cluster\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statistics import statistics from old cluster (default)\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 0cdd675e4f..3fe111fbde 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 24fcc76d72..ecf7d3d632 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,13 +141,11 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
- switch is therefore only useful to add large objects to dumps
- where a specific schema or table has been requested. Note that
- large objects are considered data and therefore will be included when
- <option>--data-only</option> is used, but not
- when <option>--schema-only</option> is.
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or <option>--statistics-only</option>, or
+ <option>--no-data</option> is specified. The <option>-b</option>
+ switch is therefore only useful to add large objects to dumps where a
+ specific schema or table has been requested.
</para>
</listitem>
</varlistentry>
@@ -516,10 +514,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +651,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -824,16 +835,17 @@ PostgreSQL documentation
<term><option>--exclude-table-data=<replaceable class="parameter">pattern</replaceable></option></term>
<listitem>
<para>
- Do not dump data for any tables matching <replaceable
+ Do not dump data or statistics for any tables matching <replaceable
class="parameter">pattern</replaceable>. The pattern is
interpreted according to the same rules as for <option>-t</option>.
<option>--exclude-table-data</option> can be given more than once to
- exclude tables matching any of several patterns. This option is
- useful when you need the definition of a particular table even
- though you do not need the data in it.
+ exclude tables matching any of several patterns. This option is useful
+ when you need the definition of a particular table even though you do
+ not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1080,6 +1092,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -1098,6 +1119,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index 39d93c2c0e..3834756973 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -265,6 +265,17 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--binary-upgrade</option></term>
<listitem>
@@ -422,6 +433,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -447,6 +467,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -456,6 +485,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719..22c3c118ad 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,20 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema (data definitions) or data.
+ </para>
+ <para>
+ (Do not confuse this with the <option>--schema</option> option, which
+ uses the word <quote>schema</quote> in a different meaning.)
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -681,6 +696,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore data, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-data-for-failed-tables</option></term>
<listitem>
@@ -713,6 +738,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema (data definitions), even if
+ the archive contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -723,6 +758,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml
index 4777381dac..64a1ebd613 100644
--- a/doc/src/sgml/ref/pgupgrade.sgml
+++ b/doc/src/sgml/ref/pgupgrade.sgml
@@ -145,6 +145,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-statistics</option></term>
+ <listitem>
+ <para>
+ Restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-o</option> <replaceable class="parameter">options</replaceable></term>
<term><option>--old-options</option> <replaceable class="parameter">options</replaceable></term>
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index a2644a2e65..3bb1e0e447 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2400,6 +2400,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelStatsInfo
RelToCheck
RelToCluster
RelabelType
--
2.48.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-06 04:45 ` Jeff Davis <[email protected]>
2025-02-06 09:35 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
0 siblings, 2 replies; 81+ messages in thread
From: Jeff Davis @ 2025-02-06 04:45 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; jian he <[email protected]>; +Cc: Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]; MichaelPaquier <[email protected]>
On Wed, 2025-02-05 at 23:01 -0500, Corey Huinker wrote:
> And here's an update to the pg_dump code itself. This currently has
> failing TAP tests for statistics in the custom and dir formats, but
> is working otherwise.
This thread got slightly mixed up, so I'm replying to the v45-0001
posted here, and also in response to Michael's and Corey's comments
from:
https://www.postgresql.org/message-id/Z5H0iRaJc1wnDVLE%40paquier.xyz
On Thu, 2025-01-23 at 16:49 +0900, Michael Paquier wrote:
> On Tue, Jan 21, 2025 at 10:21:51PM -0500, Corey Huinker wrote:
> > After some research, I think that we should treat partitioned
> > indexes like
> > we were before, and just handle the existing special case for
> > regular
> > indexes.
>
> Hmm, why? Sounds strange to me to not have the same locking
> semantics
> for the partitioned parts, and this even if partitioned indexes don't
> have stats that can be manipulated in relation_stats.c as far as I
> can see. These stats APIs are designed to be permissive as Jeff
> says.
> Having a better locking from the start makes the whole picture more
> consistent, while opening the door for actually setting real stat
> numbers for partitioned indexes (if some make sense, at some point)?
v45-0001 addresses this by locking both the partitioned index, as well
as its table, in ShareUpdateExclusive mode. That satisfies the in-place
update requirement to take a ShareUpdateExclusiveLock on the
partitioned index, while otherwise being the same as normal indexes
(and therefore unlikely to cause a problem if ANALYZE sets stats on
partitioned indexes in the future).
That means:
* For indexes: ShareUpdateExclusiveLock on table and AccessShareLock
on index
* For partitioned indexes: ShareUpdateExclusiveLock on table and
ShareUpdateExclusiveLock on index
* Otherwise, ShareupdateExclusiveLock on the relation
which makes sense to me. The v45-0001 patch itself could use some
cleanup, but I can take care of that at commit time if we agree on the
locking scheme.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-06 09:35 ` Corey Huinker <[email protected]>
1 sibling, 0 replies; 81+ messages in thread
From: Corey Huinker @ 2025-02-06 09:35 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]; MichaelPaquier <[email protected]>
>
>
> This thread got slightly mixed up, so I'm replying to the v45-0001
> posted here, and also in response to Michael's and Corey's comments
> from:
>
> https://www.postgresql.org/message-id/Z5H0iRaJc1wnDVLE%40paquier.xyz
Thanks.
>
> which makes sense to me. The v45-0001 patch itself could use some
> cleanup, but I can take care of that at commit time if we agree on the
> locking scheme.
While no changes were made to 0001 since v45, 0002 has fixed the failing
regression tests from v45. Other cleanup and a review of documentation
wording has been done, some of which I'm still not totally satisfied with,
so I'm going to give it another look tomorrow, but am putting this out for
reviewers in the mean time.
Attachments:
[text/x-patch] v46-0001-Lock-table-first-when-setting-index-relation-sta.patch (8.2K, ../../CADkLM=f5+5FBs7dQzNaMomH2_7gBy8KqP=RW_Pz_t0KniZnr-g@mail.gmail.com/3-v46-0001-Lock-table-first-when-setting-index-relation-sta.patch)
download | inline diff:
From fb095b7ea75ef366eaae5eb7b5322b9869a760ea Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Tue, 21 Jan 2025 11:52:58 -0500
Subject: [PATCH v46 1/2] Lock table first when setting index relation
statistics.
Jian He reported [1] a missing lock relation bug in
pg_restore_relation_stats when restoring stats to an index.
This fix follows the steps for proper locking prior to an inplace update
of a relation as specified in aac2c9b4fde8, and mimics the locking
behavior of the analyze command, as well as correctly doing the ACL
checks against the underlying relation of an index rather than the index
itself.
There is no special case for partitioned indexes, so while we want to
the ACL checks against the underlying relation, we need to take out
the more restrictive ShareUpdateExclusiveLock on the partitioned index.
[1] https://www.postgresql.org/message-id/CACJufxGreTY7qsCV8%2BBkuv0p5SXGTScgh%3DD%2BDq6%3D%2B_%3DXTp7FWg%40mail.gmail.com
---
src/backend/statistics/stat_utils.c | 45 +++++++++++++++---
src/test/regress/expected/stats_import.out | 53 ++++++++++++++++++++++
src/test/regress/sql/stats_import.sql | 36 +++++++++++++++
3 files changed, 128 insertions(+), 6 deletions(-)
diff --git a/src/backend/statistics/stat_utils.c b/src/backend/statistics/stat_utils.c
index 0d446f55b0..f87007e72c 100644
--- a/src/backend/statistics/stat_utils.c
+++ b/src/backend/statistics/stat_utils.c
@@ -17,13 +17,16 @@
#include "postgres.h"
#include "access/relation.h"
+#include "catalog/index.h"
#include "catalog/pg_database.h"
#include "funcapi.h"
#include "miscadmin.h"
#include "statistics/stat_utils.h"
+#include "storage/lmgr.h"
#include "utils/acl.h"
#include "utils/array.h"
#include "utils/builtins.h"
+#include "utils/lsyscache.h"
#include "utils/rel.h"
/*
@@ -126,18 +129,45 @@ stats_check_arg_pair(FunctionCallInfo fcinfo,
void
stats_lock_check_privileges(Oid reloid)
{
- Relation rel = relation_open(reloid, ShareUpdateExclusiveLock);
- const char relkind = rel->rd_rel->relkind;
+ Relation rel;
+ Oid relation_oid = reloid;
+ Oid index_oid = InvalidOid;
+ LOCKMODE index_lockmode = AccessShareLock;
- /* All of the types that can be used with ANALYZE, plus indexes */
- switch (relkind)
+ /*
+ * For indexes, we follow what do_analyze_rel() does so as to avoid any
+ * deadlocks with analyze/vacuum, which is to take out a
+ * ShareUpdateExclusive on table/matview first and then take an
+ * AccessShareLock on the index itself. See check_inplace_rel_lock()
+ * to see how this special case is implemented.
+ *
+ * Partitioned indexes do not have an exception in check_inplace_rel_lock(),
+ * so we want to take a ShareUpdateExclusive lock there instead.
+ */
+ switch(get_rel_relkind(reloid))
{
- case RELKIND_RELATION:
case RELKIND_INDEX:
+ relation_oid = IndexGetRelation(reloid, false);
+ index_oid = reloid;
+ break;
+ case RELKIND_PARTITIONED_INDEX:
+ relation_oid = IndexGetRelation(reloid, false);
+ index_oid = reloid;
+ index_lockmode = ShareUpdateExclusiveLock;
+ break;
+ default:
+ break;
+ }
+
+ rel = relation_open(relation_oid, ShareUpdateExclusiveLock);
+
+ switch (rel->rd_rel->relkind)
+ {
+ /* All of the types that can be used with ANALYZE */
+ case RELKIND_RELATION:
case RELKIND_MATVIEW:
case RELKIND_FOREIGN_TABLE:
case RELKIND_PARTITIONED_TABLE:
- case RELKIND_PARTITIONED_INDEX:
break;
default:
ereport(ERROR,
@@ -164,6 +194,9 @@ stats_lock_check_privileges(Oid reloid)
NameStr(rel->rd_rel->relname));
}
+ if (OidIsValid(index_oid))
+ LockRelationOid(index_oid, index_lockmode);
+
relation_close(rel, NoLock);
}
diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out
index fb50da1cd8..6cd584da68 100644
--- a/src/test/regress/expected/stats_import.out
+++ b/src/test/regress/expected/stats_import.out
@@ -85,6 +85,26 @@ WHERE oid = 'stats_import.test'::regclass;
17 | 400 | 4
(1 row)
+CREATE INDEX test_i ON stats_import.test(id);
+-- regular indexes have special case locking rules
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test_i'::regclass,
+ relpages => 18::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.test_i'::regclass,
+ 'relpages', 19::integer );
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- positional arguments
SELECT
pg_catalog.pg_set_relation_stats(
@@ -182,6 +202,7 @@ CREATE TABLE stats_import.part_child_1
PARTITION OF stats_import.part_parent
FOR VALUES FROM (0) TO (10)
WITH (autovacuum_enabled = false);
+CREATE INDEX part_parent_i ON stats_import.part_parent(i);
ANALYZE stats_import.part_parent;
SELECT relpages
FROM pg_class
@@ -202,6 +223,25 @@ SELECT
(1 row)
+-- Partitioned indexes aren't analyzed but it is possible to set stats.
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.part_parent_i'::regclass,
+ 'relpages', 2::integer);
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- nothing stops us from setting it to -1
SELECT
pg_catalog.pg_set_relation_stats(
@@ -1414,6 +1454,19 @@ SELECT 3, 'tre', (3, 3.3, 'TRE', '2003-03-03', NULL)::stats_import.complex_type,
UNION ALL
SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+-- Test for proper locking
+SELECT * FROM pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.is_odd'::regclass,
+ 'version', '180000'::integer,
+ 'relpages', '11'::integer,
+ 'reltuples', '10000'::real,
+ 'relallvisible', '0'::integer
+);
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- Generate statistics on table with data
ANALYZE stats_import.test;
CREATE TABLE stats_import.test_clone ( LIKE stats_import.test )
diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql
index d3058bf8f6..23e85fc6ba 100644
--- a/src/test/regress/sql/stats_import.sql
+++ b/src/test/regress/sql/stats_import.sql
@@ -64,6 +64,19 @@ SELECT relpages, reltuples, relallvisible
FROM pg_class
WHERE oid = 'stats_import.test'::regclass;
+CREATE INDEX test_i ON stats_import.test(id);
+
+-- regular indexes have special case locking rules
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test_i'::regclass,
+ relpages => 18::integer);
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.test_i'::regclass,
+ 'relpages', 19::integer );
+
-- positional arguments
SELECT
pg_catalog.pg_set_relation_stats(
@@ -127,6 +140,8 @@ CREATE TABLE stats_import.part_child_1
FOR VALUES FROM (0) TO (10)
WITH (autovacuum_enabled = false);
+CREATE INDEX part_parent_i ON stats_import.part_parent(i);
+
ANALYZE stats_import.part_parent;
SELECT relpages
@@ -140,6 +155,17 @@ SELECT
relation => 'stats_import.part_parent'::regclass,
relpages => 2::integer);
+-- Partitioned indexes aren't analyzed but it is possible to set stats.
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.part_parent_i'::regclass,
+ 'relpages', 2::integer);
+
-- nothing stops us from setting it to -1
SELECT
pg_catalog.pg_set_relation_stats(
@@ -1062,6 +1088,16 @@ SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+
+-- Test for proper locking
+SELECT * FROM pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.is_odd'::regclass,
+ 'version', '180000'::integer,
+ 'relpages', '11'::integer,
+ 'reltuples', '10000'::real,
+ 'relallvisible', '0'::integer
+);
+
-- Generate statistics on table with data
ANALYZE stats_import.test;
base-commit: 9e020050b8fa8e184bc1d58e6a4bc1edfa76cb8c
--
2.48.1
[text/x-patch] v46-0002-Enable-dumping-of-table-index-stats-in-pg_dump.patch (66.7K, ../../CADkLM=f5+5FBs7dQzNaMomH2_7gBy8KqP=RW_Pz_t0KniZnr-g@mail.gmail.com/4-v46-0002-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From 983d67a266ce3f66df7e8e8579db38b13eaf9816 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v46 2/2] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute that
has a corresponding row in pg_statistic. These statements will restore
the statistics of the current system onto the destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, statistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index statistics are dumped in the
post-data section.
Add --no-data option. This option is useful for situations where
someone wishes to test query plans from a production database without
copying production data.
This also makes the corresponding change to the simulated pg_upgrade in
the TAP tests for pg_dump.
Add --no-schema option to pg_dump, etc. Previously, users could use
--data-only when they wanted to suppress schema from a dump. However,
that no longer makes sense now that the data/schema binary has become
the data/schema/statistics trinary.
---
src/bin/pg_dump/pg_backup.h | 10 +-
src/bin/pg_dump/pg_backup_archiver.c | 97 +++++--
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_backup_directory.c | 2 +-
src/bin/pg_dump/pg_dump.c | 390 +++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 11 +
src/bin/pg_dump/pg_dump_sort.c | 36 ++-
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 31 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
src/bin/pg_dump/t/002_pg_dump.pl | 111 +++++++-
src/bin/pg_upgrade/dump.c | 6 +-
src/bin/pg_upgrade/option.c | 12 +
src/bin/pg_upgrade/pg_upgrade.h | 1 +
doc/src/sgml/ref/pg_dump.sgml | 69 ++++-
doc/src/sgml/ref/pg_dumpall.sgml | 38 +++
doc/src/sgml/ref/pg_restore.sgml | 47 +++-
doc/src/sgml/ref/pgupgrade.sgml | 18 ++
src/tools/pgindent/typedefs.list | 1 +
19 files changed, 841 insertions(+), 65 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b2..3fa1474fad 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,9 +110,12 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +163,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -179,8 +183,11 @@ typedef struct _dumpOptions
int column_inserts;
int if_exists;
int no_comments;
- int no_security_labels;
+ int no_data;
int no_publications;
+ int no_schema;
+ int no_security_labels;
+ int no_statistics;
int no_subscriptions;
int no_toast_compression;
int no_unlogged_table_data;
@@ -208,6 +215,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844..cfa1349b8a 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -46,6 +46,12 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
+typedef enum entryType
+{
+ default_entry,
+ data_entry,
+ statistics_entry
+} entryType;
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -53,7 +59,7 @@ static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr,
DataDirSyncMethod sync_method);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
-static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
+static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type);
static char *sanitize_line(const char *str, bool want_hyphen);
static void _doSetFixedOutputState(ArchiveHandle *AH);
static void _doSetSessionAuth(ArchiveHandle *AH, const char *user);
@@ -149,6 +155,7 @@ InitDumpOptions(DumpOptions *opts)
opts->dumpSections = DUMP_UNSECTIONED;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
}
/*
@@ -169,9 +176,10 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->outputClean = ropt->dropSchema;
dopt->dumpData = ropt->dumpData;
dopt->dumpSchema = ropt->dumpSchema;
+ dopt->dumpSections = ropt->dumpSections;
+ dopt->dumpStatistics = ropt->dumpStatistics;
dopt->if_exists = ropt->if_exists;
dopt->column_inserts = ropt->column_inserts;
- dopt->dumpSections = ropt->dumpSections;
dopt->aclsSkip = ropt->aclsSkip;
dopt->outputSuperuser = ropt->superuser;
dopt->outputCreateDB = ropt->createDB;
@@ -186,6 +194,9 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
@@ -418,30 +429,30 @@ RestoreArchive(Archive *AHX)
}
/*
- * Work out if we have an implied data-only restore. This can happen if
- * the dump was data only or if the user has used a toc list to exclude
- * all of the schema data. All we do is look for schema entries - if none
- * are found then we unset the dumpSchema flag.
+ * Work out if we have an schema-less restore. This can happen if the dump
+ * was data-only or statistics-only or no-schema or if the user has used a
+ * toc list to exclude all of the schema data. All we do is look for
+ * schema entries - if none are found then we unset the dumpSchema flag.
*
* We could scan for wanted TABLE entries, but that is not the same as
* data-only. At this stage, it seems unnecessary (6-Mar-2001).
*/
if (ropt->dumpSchema)
{
- int impliedDataOnly = 1;
+ bool no_schema_found = true;
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if ((te->reqs & REQ_SCHEMA) != 0)
- { /* It's schema, and it's wanted */
- impliedDataOnly = 0;
+ {
+ no_schema_found = false;
break;
}
}
- if (impliedDataOnly)
+ if (no_schema_found)
{
ropt->dumpSchema = false;
- pg_log_info("implied data-only restore");
+ pg_log_info("implied no-schema restore");
}
}
@@ -739,7 +750,7 @@ RestoreArchive(Archive *AHX)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
continue; /* ignore if not to be dumped at all */
switch (_tocEntryRestorePass(te))
@@ -760,7 +771,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -770,7 +781,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_POST_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -869,7 +880,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
pg_log_info("creating %s \"%s\"",
te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
defnDumped = true;
if (strcmp(te->desc, "TABLE") == 0)
@@ -938,7 +949,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
*/
if (AH->PrintTocDataPtr != NULL)
{
- _printTocEntry(AH, te, true);
+ _printTocEntry(AH, te, data_entry);
if (strcmp(te->desc, "BLOBS") == 0 ||
strcmp(te->desc, "BLOB COMMENTS") == 0)
@@ -1036,15 +1047,21 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
{
/* If we haven't already dumped the defn part, do so now */
pg_log_info("executing %s %s", te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
}
}
+ /*
+ * If it has a statistics component that we want, then process that
+ */
+ if ((reqs & REQ_STATS) != 0)
+ _printTocEntry(AH, te, statistics_entry);
+
/*
* If we emitted anything for this TOC entry, that counts as one action
* against the transaction-size limit. Commit if it's time to.
*/
- if ((reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && ropt->txn_size > 0)
+ if ((reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && ropt->txn_size > 0)
{
if (++AH->txnCount >= ropt->txn_size)
{
@@ -1084,6 +1101,7 @@ NewRestoreOptions(void)
opts->compression_spec.level = 0;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
return opts;
}
@@ -1329,7 +1347,7 @@ PrintTOCSummary(Archive *AHX)
te->reqs = _tocEntryRequired(te, curSection, AH);
/* Now, should we print it? */
if (ropt->verbose ||
- (te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0)
+ (te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0)
{
char *sanitized_name;
char *sanitized_schema;
@@ -2582,7 +2600,7 @@ WriteToc(ArchiveHandle *AH)
tocCount = 0;
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) != 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) != 0)
tocCount++;
}
@@ -2592,7 +2610,7 @@ WriteToc(ArchiveHandle *AH)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) == 0)
continue;
WriteInt(AH, te->dumpId);
@@ -2918,6 +2936,14 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strcmp(te->desc, "SEARCHPATH") == 0)
return REQ_SPECIAL;
+ if (strcmp(te->desc, "STATISTICS DATA") == 0)
+ {
+ if (!ropt->dumpStatistics)
+ return 0;
+ else
+ res = REQ_STATS;
+ }
+
/*
* DATABASE and DATABASE PROPERTIES also have a special rule: they are
* restored in createDB mode, and not restored otherwise, independently of
@@ -2962,6 +2988,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's statistics and we don't want statistics, maybe ignore it */
+ if (!ropt->dumpStatistics && strcmp(te->desc, "STATISTICS DATA") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +3021,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS DATA") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
@@ -3107,6 +3138,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
}
+
/*
* Determine whether the TOC entry contains schema and/or data components,
* and mask off inapplicable REQ bits. If it had a dataDumper, assume
@@ -3172,12 +3204,12 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strncmp(te->tag, "LARGE OBJECT", 12) == 0) ||
(strcmp(te->desc, "SECURITY LABEL") == 0 &&
strncmp(te->tag, "LARGE OBJECT", 12) == 0))))
- res = res & REQ_SCHEMA;
+ res = res & (REQ_SCHEMA | REQ_STATS);
}
/* Mask it if we don't want schema */
if (!ropt->dumpSchema)
- res = res & REQ_DATA;
+ res = res & (REQ_DATA | REQ_STATS);
return res;
}
@@ -3729,7 +3761,7 @@ _getObjectDescription(PQExpBuffer buf, const TocEntry *te)
* will remain at default, until the matching ACL TOC entry is restored.
*/
static void
-_printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
+_printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type)
{
RestoreOptions *ropt = AH->public.ropt;
@@ -3753,10 +3785,17 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
char *sanitized_schema;
char *sanitized_owner;
- if (isData)
- pfx = "Data for ";
- else
- pfx = "";
+ switch (entry_type)
+ {
+ case data_entry:
+ pfx = "Data for ";
+ break;
+ case statistics_entry:
+ pfx = "Statistics for ";
+ break;
+ default:
+ pfx = "";
+ }
ahprintf(AH, "--\n");
if (AH->public.verbose)
@@ -4324,7 +4363,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
- if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
{
pg_log_info("skipping item %d %s %s",
next_work_item->dumpId,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index ce5ed1dd39..a2064f471e 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -209,7 +209,8 @@ typedef enum
#define REQ_SCHEMA 0x01 /* want schema */
#define REQ_DATA 0x02 /* want data */
-#define REQ_SPECIAL 0x04 /* for special TOC entries */
+#define REQ_STATS 0x04
+#define REQ_SPECIAL 0x08 /* for special TOC entries */
struct _archiveHandle
{
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 240a1d4106..b2a841bb0f 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -780,7 +780,7 @@ _PrepParallelRestore(ArchiveHandle *AH)
continue;
/* We may ignore items not due to be restored */
- if ((te->reqs & REQ_DATA) == 0)
+ if ((te->reqs & (REQ_DATA | REQ_STATS)) == 0)
continue;
/*
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 02e1fdf8f7..c92a31902a 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -431,6 +431,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -467,6 +468,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -493,8 +495,11 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -540,7 +545,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -614,6 +619,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -785,6 +794,17 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -799,8 +819,9 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
@@ -1100,6 +1121,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1178,7 +1200,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1191,11 +1213,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1220,8 +1243,11 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
+ printf(_(" --no-schema do not dump schema\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6778,6 +6804,43 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+ info->postponed_def = false;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7203,6 +7266,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting || dopt->dumpStatistics)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7649,11 +7714,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7676,7 +7744,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7710,6 +7785,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10297,6 +10374,296 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * Decide which section to use based on the relkind of the parent object.
+ *
+ * NB: materialized views may be postponed from SECTION_PRE_DATA to
+ * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the
+ * matview stats will also be postponed to SECTION_POST_DATA. See
+ * repairMatViewBoundaryMultiLoop().
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+ switch (rsinfo->relkind)
+ {
+ case RELKIND_RELATION:
+ case RELKIND_PARTITIONED_TABLE:
+ case RELKIND_MATVIEW:
+ return SECTION_DATA;
+ case RELKIND_INDEX:
+ case RELKIND_PARTITIONED_INDEX:
+ return SECTION_POST_DATA;
+ default:
+ pg_fatal("cannot dump statistics for relation kind '%c'",
+ rsinfo->relkind);
+ }
+
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+ DumpId *deps = NULL;
+ int ndeps = 0;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ /* dependent on the relation definition, if doing schema */
+ if (fout->dopt->dumpSchema)
+ {
+ deps = dobj->dependencies;
+ ndeps = dobj->nDeps;
+ }
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = rsinfo->postponed_def ?
+ SECTION_POST_DATA : statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = deps,
+ .nDeps = ndeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10745,6 +11112,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -18971,6 +19341,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 7139c88a69..410c162a85 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -83,10 +83,13 @@ typedef enum
DO_PUBLICATION,
DO_PUBLICATION_REL,
DO_PUBLICATION_TABLE_IN_SCHEMA,
+ DO_REL_STATS,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
} DumpableObjectType;
+#define NUM_DUMPABLE_OBJECT_TYPES (DO_SUBSCRIPTION_REL + 1)
+
/*
* DumpComponents is a bitmask of the potentially dumpable components of
* a database object: its core definition, plus optional attributes such
@@ -110,6 +113,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -430,6 +434,13 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'm', 'i', etc */
+ bool postponed_def; /* stats must be postponed into post-data */
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index dc9a28924b..201eeedde7 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -81,6 +81,7 @@ enum dbObjectTypePriorities
PRIO_TABLE_DATA,
PRIO_SEQUENCE_SET,
PRIO_LARGE_OBJECT_DATA,
+ PRIO_STATISTICS_DATA_DATA,
PRIO_POST_DATA_BOUNDARY, /* boundary! */
PRIO_CONSTRAINT,
PRIO_INDEX,
@@ -148,11 +149,12 @@ static const int dbObjectTypePriority[] =
[DO_PUBLICATION] = PRIO_PUBLICATION,
[DO_PUBLICATION_REL] = PRIO_PUBLICATION_REL,
[DO_PUBLICATION_TABLE_IN_SCHEMA] = PRIO_PUBLICATION_TABLE_IN_SCHEMA,
+ [DO_REL_STATS] = PRIO_STATISTICS_DATA_DATA,
[DO_SUBSCRIPTION] = PRIO_SUBSCRIPTION,
[DO_SUBSCRIPTION_REL] = PRIO_SUBSCRIPTION_REL,
};
-StaticAssertDecl(lengthof(dbObjectTypePriority) == (DO_SUBSCRIPTION_REL + 1),
+StaticAssertDecl(lengthof(dbObjectTypePriority) == NUM_DUMPABLE_OBJECT_TYPES,
"array length mismatch");
static DumpId preDataBoundId;
@@ -801,11 +803,21 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj,
{
/* remove boundary's dependency on object after it in loop */
removeObjectDependency(boundaryobj, nextobj->dumpId);
- /* if that object is a matview, mark it as postponed into post-data */
+ /*
+ * If that object is a matview or matview status, mark it as postponed into
+ * post-data.
+ */
if (nextobj->objType == DO_TABLE)
{
TableInfo *nextinfo = (TableInfo *) nextobj;
+ if (nextinfo->relkind == RELKIND_MATVIEW)
+ nextinfo->postponed_def = true;
+ }
+ else if (nextobj->objType == DO_REL_STATS)
+ {
+ RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj;
+
if (nextinfo->relkind == RELKIND_MATVIEW)
nextinfo->postponed_def = true;
}
@@ -1018,6 +1030,21 @@ repairDependencyLoop(DumpableObject **loop,
{
DumpableObject *nextobj;
+ nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
+ repairMatViewBoundaryMultiLoop(loop[j], nextobj);
+ return;
+ }
+ }
+ }
+ else if (loop[i]->objType == DO_REL_STATS &&
+ ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW)
+ {
+ for (j = 0; j < nLoop; j++)
+ {
+ if (loop[j]->objType == DO_POST_DATA_BOUNDARY)
+ {
+ DumpableObject *nextobj;
+
nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
repairMatViewBoundaryMultiLoop(loop[j], nextobj);
return;
@@ -1500,6 +1527,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 396f79781c..7effb70490 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index c602272d7d..02af89bae1 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,9 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool data_only = false;
+ bool schema_only = false;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -71,12 +74,13 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_comments = 0;
+ static int no_data = 0;
static int no_publications = 0;
+ static int no_schema = 0;
static int no_security_labels = 0;
+ static int no_statistics = 0;
static int no_subscriptions = 0;
static int strict_names = 0;
- bool data_only = false;
- bool schema_only = false;
struct option cmdopts[] = {
{"clean", 0, NULL, 'c'},
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -124,9 +129,12 @@ main(int argc, char **argv)
{"transaction-size", required_argument, NULL, 5},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
@@ -271,6 +279,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +355,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +378,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpData = data_only || (!no_data && !schema_only && !statistics_only);
+ opts->dumpSchema = schema_only || (!no_schema && !data_only && !statistics_only);
+ opts->dumpStatistics = statistics_only || (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +392,8 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
@@ -482,6 +501,7 @@ usage(const char *progname)
printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n"));
printf(_(" -T, --trigger=NAME restore named trigger\n"));
printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only restore only the statistics, not schema or data\n"));
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
@@ -489,10 +509,13 @@ usage(const char *progname)
" in FILENAME\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comment commands\n"));
+ printf(_(" --no-data do not restore data\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 214240f1ae..f29da06ed2 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 805ba9f49f..d5f0f92dfa 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -65,7 +65,7 @@ my %pgdump_runs = (
'--format' => 'custom',
'--file' => "$tempdir/binary_upgrade.dump",
'--no-password',
- '--schema-only',
+ '--no-data',
'--binary-upgrade',
'--dbname' => 'postgres', # alternative way to specify database
],
@@ -710,6 +710,39 @@ my %pgdump_runs = (
'--no-large-objects',
'postgres',
],
+ },
+ no_statistics => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_statistics.sql",
+ '--no-statistics',
+ 'postgres',
+ ],
+ },
+ no_data_no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_data_no_schema.sql",
+ '--no-data',
+ '--no-schema',
+ 'postgres',
+ ],
+ },
+ statistics_only => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/statistics_only.sql",
+ '--statistics-only',
+ 'postgres',
+ ],
+ },
+ no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_schema.sql",
+ '--no-schema',
+ 'postgres',
+ ],
},);
###############################################################
@@ -776,6 +809,7 @@ my %full_runs = (
no_large_objects => 1,
no_owner => 1,
no_privs => 1,
+ no_statistics => 1,
no_table_access_method => 1,
pg_dumpall_dbprivs => 1,
pg_dumpall_exclude => 1,
@@ -977,6 +1011,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1390,6 +1425,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1411,6 +1447,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1432,6 +1469,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1598,6 +1636,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1751,6 +1790,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_test_table => 1,
section_data => 1,
},
@@ -1778,6 +1818,7 @@ my %tests = (
data_only => 1,
exclude_test_table => 1,
exclude_test_table_data => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1798,7 +1839,10 @@ my %tests = (
\QCOPY dump_test.fk_reference_test_table (col1) FROM stdin;\E
\n(?:\d\n){5}\\\.\n
/xms,
- like => { data_only => 1, },
+ like => {
+ data_only => 1,
+ no_schema => 1,
+ },
},
'COPY test_second_table' => {
@@ -1814,6 +1858,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1836,6 +1881,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1859,6 +1905,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1881,6 +1928,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1903,6 +1951,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -3299,6 +3348,7 @@ my %tests = (
like => {
%full_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
only_dump_test_schema => 1,
test_schema_plus_large_objects => 1,
@@ -3469,6 +3519,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_measurement => 1,
section_data => 1,
only_dump_test_schema => 1,
@@ -4351,6 +4402,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
binary_upgrade => 1,
@@ -4651,6 +4703,61 @@ my %tests = (
},
},
+ #
+ # TABLE and MATVIEW stats will end up in SECTION_DATA.
+ # INDEX stats (expression columns only) will end up in SECTION_POST_DATA.
+ #
+ 'statistics_import' => {
+ create_sql => '
+ CREATE TABLE dump_test.has_stats
+ AS SELECT g.g AS x, g.g / 2 AS y FROM generate_series(1,100) AS g(g);
+ CREATE MATERIALIZED VIEW dump_test.has_stats_mv AS SELECT * FROM dump_test.has_stats;
+ CREATE INDEX dup_test_post_data_ix ON dump_test.has_stats((x - 1));
+ ANALYZE dump_test.has_stats, dump_test.has_stats_mv;',
+ regexp => qr/pg_catalog.pg_restore_attribute_stats/,
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ exclude_dump_test_schema => 1,
+ no_statistics => 1,
+ only_dump_measurement => 1,
+ schema_only => 1,
+ },
+ },
+
+ #
+ # While attribute stats (aka pg_statistic stats) only appear for tables
+ # that have been analyzed, all tables will have relation stats because
+ # those come from pg_class.
+ #
+ 'relstats_on_unanalyzed_tables' => {
+ regexp => qr/pg_catalog.pg_restore_relation_stats/,
+
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ only_dump_test_table => 1,
+ role => 1,
+ role_parallel => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ no_statistics => 1,
+ schema_only => 1,
+ },
+ },
+
# CREATE TABLE with partitioned table and various AMs. One
# partition uses the same default as the parent, and a second
# uses its own AM.
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8ce0fa3020..a29cd2cca9 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 108eb7a1ba..3b6c7ec994 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -306,7 +316,9 @@ usage(void)
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
+ printf(_(" --no-statistics do not import statistics from old cluster\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statistics import statistics from old cluster (default)\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 0cdd675e4f..3fe111fbde 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 24fcc76d72..ecf7d3d632 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,13 +141,11 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
- switch is therefore only useful to add large objects to dumps
- where a specific schema or table has been requested. Note that
- large objects are considered data and therefore will be included when
- <option>--data-only</option> is used, but not
- when <option>--schema-only</option> is.
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or <option>--statistics-only</option>, or
+ <option>--no-data</option> is specified. The <option>-b</option>
+ switch is therefore only useful to add large objects to dumps where a
+ specific schema or table has been requested.
</para>
</listitem>
</varlistentry>
@@ -516,10 +514,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +651,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -824,16 +835,17 @@ PostgreSQL documentation
<term><option>--exclude-table-data=<replaceable class="parameter">pattern</replaceable></option></term>
<listitem>
<para>
- Do not dump data for any tables matching <replaceable
+ Do not dump data or statistics for any tables matching <replaceable
class="parameter">pattern</replaceable>. The pattern is
interpreted according to the same rules as for <option>-t</option>.
<option>--exclude-table-data</option> can be given more than once to
- exclude tables matching any of several patterns. This option is
- useful when you need the definition of a particular table even
- though you do not need the data in it.
+ exclude tables matching any of several patterns. This option is useful
+ when you need the definition of a particular table even though you do
+ not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1080,6 +1092,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -1098,6 +1119,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index 39d93c2c0e..3834756973 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -265,6 +265,17 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--binary-upgrade</option></term>
<listitem>
@@ -422,6 +433,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -447,6 +467,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -456,6 +485,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719..3c381db1aa 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema (data definitions) or data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -681,6 +692,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore data, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-data-for-failed-tables</option></term>
<listitem>
@@ -713,6 +734,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema (data definitions), even if
+ the archive contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -723,6 +754,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml
index 4777381dac..64a1ebd613 100644
--- a/doc/src/sgml/ref/pgupgrade.sgml
+++ b/doc/src/sgml/ref/pgupgrade.sgml
@@ -145,6 +145,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-statistics</option></term>
+ <listitem>
+ <para>
+ Restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-o</option> <replaceable class="parameter">options</replaceable></term>
<term><option>--old-options</option> <replaceable class="parameter">options</replaceable></term>
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9a3bee93de..f3351342a2 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2400,6 +2400,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelStatsInfo
RelToCheck
RelToCluster
RelabelType
--
2.48.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-07 06:07 ` Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Michael Paquier @ 2025-02-07 06:07 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Corey Huinker <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Wed, Feb 05, 2025 at 08:45:06PM -0800, Jeff Davis wrote:
> v45-0001 addresses this by locking both the partitioned index, as well
> as its table, in ShareUpdateExclusive mode. That satisfies the in-place
> update requirement to take a ShareUpdateExclusiveLock on the
> partitioned index, while otherwise being the same as normal indexes
> (and therefore unlikely to cause a problem if ANALYZE sets stats on
> partitioned indexes in the future).
>
> That means:
> * For indexes: ShareUpdateExclusiveLock on table and AccessShareLock
> on index
> * For partitioned indexes: ShareUpdateExclusiveLock on table and
> ShareUpdateExclusiveLock on index
> * Otherwise, ShareupdateExclusiveLock on the relation
>
> which makes sense to me. The v45-0001 patch itself could use some
> cleanup, but I can take care of that at commit time if we agree on the
> locking scheme.
Fine by me.
The regression tests of v45 and v46 are a bit fuzzy regarding the
tests around locking for partitioned tables. For example, with v46
applied on top of HEAD, if I manipulate the internals of the patch in
stats_check_arg_pair() so as we don't take a lock on the parent table,
then a make check is still happy and passes even if the internals are
clearly broken.
I would recommend to work a bit more the tests by updating the stats
of a relation with the SQL functions in a transaction and add some
queries on pg_locks for locktype = 'relation' that are able to check
the locks we are taking when running these operations (return pairs of
relation::regclass and mode, for example). Doing that for
non-partitioned relations is also something I would do, so as the
locking schema we are using is clearly tracked and that future
manipulations of the area would help one in tracking problems. Bonus
points: scans of pg_locks are cheap tests.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
@ 2025-02-10 03:00 ` Corey Huinker <[email protected]>
2025-02-10 23:36 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 2 replies; 81+ messages in thread
From: Corey Huinker @ 2025-02-10 03:00 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Jeff Davis <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
> I would recommend to work a bit more the tests by updating the stats
> of a relation with the SQL functions in a transaction and add some
> queries on pg_locks for locktype = 'relation' that are able to check
> the locks we are taking when running these operations (return pairs of
> relation::regclass and mode, for example). Doing that for
> non-partitioned relations is also something I would do, so as the
> locking schema we are using is clearly tracked and that future
> manipulations of the area would help one in tracking problems. Bonus
> points: scans of pg_locks are cheap tests.
> --
> Michael
>
0001 - I've added pg_locks tests for a regular index and a partitioned
index.
0002 - I've done some documentation rewording, mostly wording changes where
behaviors surrounding data-only dumps are actually meant for any dump that
has all schema excluded.
Attachments:
[text/x-patch] v47-0001-Lock-table-first-when-setting-index-relation-sta.patch (10.4K, ../../CADkLM=fzX7QX6r78fShWDjNN3Vcr4PVAnvXxQ4DiGy6V=0bCUA@mail.gmail.com/3-v47-0001-Lock-table-first-when-setting-index-relation-sta.patch)
download | inline diff:
From 2de404f8187466061469abfaf3a773290cce6af8 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Tue, 21 Jan 2025 11:52:58 -0500
Subject: [PATCH v47 1/2] Lock table first when setting index relation
statistics.
Jian He reported [1] a missing lock relation bug in
pg_restore_relation_stats when restoring stats to an index.
This fix follows the steps for proper locking prior to an inplace update
of a relation as specified in aac2c9b4fde8, and mimics the locking
behavior of the analyze command, as well as correctly doing the ACL
checks against the underlying relation of an index rather than the index
itself.
There is no special case for partitioned indexes, so while we want to
the ACL checks against the underlying relation, we need to take out
the more restrictive ShareUpdateExclusiveLock on the partitioned index.
[1] https://www.postgresql.org/message-id/CACJufxGreTY7qsCV8%2BBkuv0p5SXGTScgh%3DD%2BDq6%3D%2B_%3DXTp7FWg%40mail.gmail.com
---
src/backend/statistics/stat_utils.c | 45 +++++++--
src/test/regress/expected/stats_import.out | 106 +++++++++++++++++++++
src/test/regress/sql/stats_import.sql | 72 ++++++++++++++
3 files changed, 217 insertions(+), 6 deletions(-)
diff --git a/src/backend/statistics/stat_utils.c b/src/backend/statistics/stat_utils.c
index 0d446f55b01..f87007e72c2 100644
--- a/src/backend/statistics/stat_utils.c
+++ b/src/backend/statistics/stat_utils.c
@@ -17,13 +17,16 @@
#include "postgres.h"
#include "access/relation.h"
+#include "catalog/index.h"
#include "catalog/pg_database.h"
#include "funcapi.h"
#include "miscadmin.h"
#include "statistics/stat_utils.h"
+#include "storage/lmgr.h"
#include "utils/acl.h"
#include "utils/array.h"
#include "utils/builtins.h"
+#include "utils/lsyscache.h"
#include "utils/rel.h"
/*
@@ -126,18 +129,45 @@ stats_check_arg_pair(FunctionCallInfo fcinfo,
void
stats_lock_check_privileges(Oid reloid)
{
- Relation rel = relation_open(reloid, ShareUpdateExclusiveLock);
- const char relkind = rel->rd_rel->relkind;
+ Relation rel;
+ Oid relation_oid = reloid;
+ Oid index_oid = InvalidOid;
+ LOCKMODE index_lockmode = AccessShareLock;
- /* All of the types that can be used with ANALYZE, plus indexes */
- switch (relkind)
+ /*
+ * For indexes, we follow what do_analyze_rel() does so as to avoid any
+ * deadlocks with analyze/vacuum, which is to take out a
+ * ShareUpdateExclusive on table/matview first and then take an
+ * AccessShareLock on the index itself. See check_inplace_rel_lock()
+ * to see how this special case is implemented.
+ *
+ * Partitioned indexes do not have an exception in check_inplace_rel_lock(),
+ * so we want to take a ShareUpdateExclusive lock there instead.
+ */
+ switch(get_rel_relkind(reloid))
{
- case RELKIND_RELATION:
case RELKIND_INDEX:
+ relation_oid = IndexGetRelation(reloid, false);
+ index_oid = reloid;
+ break;
+ case RELKIND_PARTITIONED_INDEX:
+ relation_oid = IndexGetRelation(reloid, false);
+ index_oid = reloid;
+ index_lockmode = ShareUpdateExclusiveLock;
+ break;
+ default:
+ break;
+ }
+
+ rel = relation_open(relation_oid, ShareUpdateExclusiveLock);
+
+ switch (rel->rd_rel->relkind)
+ {
+ /* All of the types that can be used with ANALYZE */
+ case RELKIND_RELATION:
case RELKIND_MATVIEW:
case RELKIND_FOREIGN_TABLE:
case RELKIND_PARTITIONED_TABLE:
- case RELKIND_PARTITIONED_INDEX:
break;
default:
ereport(ERROR,
@@ -164,6 +194,9 @@ stats_lock_check_privileges(Oid reloid)
NameStr(rel->rd_rel->relname));
}
+ if (OidIsValid(index_oid))
+ LockRelationOid(index_oid, index_lockmode);
+
relation_close(rel, NoLock);
}
diff --git a/src/test/regress/expected/stats_import.out b/src/test/regress/expected/stats_import.out
index fb50da1cd83..a479300a68d 100644
--- a/src/test/regress/expected/stats_import.out
+++ b/src/test/regress/expected/stats_import.out
@@ -85,6 +85,46 @@ WHERE oid = 'stats_import.test'::regclass;
17 | 400 | 4
(1 row)
+CREATE INDEX test_i ON stats_import.test(id);
+BEGIN;
+-- regular indexes have special case locking rules
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test_i'::regclass,
+ relpages => 18::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.test'::regclass
+AND granted;
+ mode
+--------------------------
+ ShareUpdateExclusiveLock
+(1 row)
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.test_i'::regclass
+AND granted;
+ mode
+-----------------
+ AccessShareLock
+(1 row)
+
+COMMIT;
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.test_i'::regclass,
+ 'relpages', 19::integer );
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- positional arguments
SELECT
pg_catalog.pg_set_relation_stats(
@@ -182,6 +222,7 @@ CREATE TABLE stats_import.part_child_1
PARTITION OF stats_import.part_parent
FOR VALUES FROM (0) TO (10)
WITH (autovacuum_enabled = false);
+CREATE INDEX part_parent_i ON stats_import.part_parent(i);
ANALYZE stats_import.part_parent;
SELECT relpages
FROM pg_class
@@ -193,6 +234,15 @@ WHERE oid = 'stats_import.part_parent'::regclass;
-- although partitioned tables have no storage, setting relpages to a
-- positive value is still allowed
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
SELECT
pg_catalog.pg_set_relation_stats(
relation => 'stats_import.part_parent'::regclass,
@@ -202,6 +252,49 @@ SELECT
(1 row)
+--
+-- Partitioned indexes aren't analyzed but it is possible to set stats.
+-- Partitioned indexes also have less relaxed locking rules than regular
+-- indexes.
+--
+BEGIN;
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+ pg_set_relation_stats
+-----------------------
+
+(1 row)
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.part_parent'::regclass
+AND granted;
+ mode
+--------------------------
+ ShareUpdateExclusiveLock
+(1 row)
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.part_parent_i'::regclass
+AND granted;
+ mode
+--------------------------
+ ShareUpdateExclusiveLock
+(1 row)
+
+COMMIT;
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.part_parent_i'::regclass,
+ 'relpages', 2::integer);
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- nothing stops us from setting it to -1
SELECT
pg_catalog.pg_set_relation_stats(
@@ -1414,6 +1507,19 @@ SELECT 3, 'tre', (3, 3.3, 'TRE', '2003-03-03', NULL)::stats_import.complex_type,
UNION ALL
SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+-- Test for proper locking
+SELECT * FROM pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.is_odd'::regclass,
+ 'version', '180000'::integer,
+ 'relpages', '11'::integer,
+ 'reltuples', '10000'::real,
+ 'relallvisible', '0'::integer
+);
+ pg_restore_relation_stats
+---------------------------
+ t
+(1 row)
+
-- Generate statistics on table with data
ANALYZE stats_import.test;
CREATE TABLE stats_import.test_clone ( LIKE stats_import.test )
diff --git a/src/test/regress/sql/stats_import.sql b/src/test/regress/sql/stats_import.sql
index d3058bf8f6b..f5b8bd5fc58 100644
--- a/src/test/regress/sql/stats_import.sql
+++ b/src/test/regress/sql/stats_import.sql
@@ -64,6 +64,32 @@ SELECT relpages, reltuples, relallvisible
FROM pg_class
WHERE oid = 'stats_import.test'::regclass;
+CREATE INDEX test_i ON stats_import.test(id);
+
+BEGIN;
+-- regular indexes have special case locking rules
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.test_i'::regclass,
+ relpages => 18::integer);
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.test'::regclass
+AND granted;
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.test_i'::regclass
+AND granted;
+
+COMMIT;
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.test_i'::regclass,
+ 'relpages', 19::integer );
+
-- positional arguments
SELECT
pg_catalog.pg_set_relation_stats(
@@ -127,6 +153,8 @@ CREATE TABLE stats_import.part_child_1
FOR VALUES FROM (0) TO (10)
WITH (autovacuum_enabled = false);
+CREATE INDEX part_parent_i ON stats_import.part_parent(i);
+
ANALYZE stats_import.part_parent;
SELECT relpages
@@ -135,11 +163,45 @@ WHERE oid = 'stats_import.part_parent'::regclass;
-- although partitioned tables have no storage, setting relpages to a
-- positive value is still allowed
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+
SELECT
pg_catalog.pg_set_relation_stats(
relation => 'stats_import.part_parent'::regclass,
relpages => 2::integer);
+--
+-- Partitioned indexes aren't analyzed but it is possible to set stats.
+-- Partitioned indexes also have less relaxed locking rules than regular
+-- indexes.
+--
+BEGIN;
+
+SELECT
+ pg_catalog.pg_set_relation_stats(
+ relation => 'stats_import.part_parent_i'::regclass,
+ relpages => 2::integer);
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.part_parent'::regclass
+AND granted;
+
+SELECT mode
+FROM pg_locks
+WHERE relation = 'stats_import.part_parent_i'::regclass
+AND granted;
+
+COMMIT;
+
+SELECT
+ pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.part_parent_i'::regclass,
+ 'relpages', 2::integer);
+
-- nothing stops us from setting it to -1
SELECT
pg_catalog.pg_set_relation_stats(
@@ -1062,6 +1124,16 @@ SELECT 4, 'four', NULL, int4range(0,100), NULL;
CREATE INDEX is_odd ON stats_import.test(((comp).a % 2 = 1));
+
+-- Test for proper locking
+SELECT * FROM pg_catalog.pg_restore_relation_stats(
+ 'relation', 'stats_import.is_odd'::regclass,
+ 'version', '180000'::integer,
+ 'relpages', '11'::integer,
+ 'reltuples', '10000'::real,
+ 'relallvisible', '0'::integer
+);
+
-- Generate statistics on table with data
ANALYZE stats_import.test;
base-commit: 3d17d7d7fb7a4603b48acb275b5a416f110db464
--
2.48.1
[text/x-patch] v47-0002-Enable-dumping-of-table-index-stats-in-pg_dump.patch (69.7K, ../../CADkLM=fzX7QX6r78fShWDjNN3Vcr4PVAnvXxQ4DiGy6V=0bCUA@mail.gmail.com/4-v47-0002-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From 6e971f1528bdb95a5de86af63ac68e8367501c87 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v47 2/2] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute that
has a corresponding row in pg_statistic. These statements will restore
the statistics of the current system onto the destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, statistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index statistics are dumped in the
post-data section.
Add --no-data option. This option is useful for situations where
someone wishes to test query plans from a production database without
copying production data.
This also makes the corresponding change to the simulated pg_upgrade in
the TAP tests for pg_dump.
Add --no-schema option to pg_dump, etc. Previously, users could use
--data-only when they wanted to suppress schema from a dump. However,
that no longer makes sense now that the data/schema binary has become
the data/schema/statistics trinary.
---
src/bin/pg_dump/pg_backup.h | 10 +-
src/bin/pg_dump/pg_backup_archiver.c | 97 +++++--
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_backup_directory.c | 2 +-
src/bin/pg_dump/pg_dump.c | 390 +++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 11 +
src/bin/pg_dump/pg_dump_sort.c | 36 ++-
src/bin/pg_dump/pg_dumpall.c | 5 +
src/bin/pg_dump/pg_restore.c | 31 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
src/bin/pg_dump/t/002_pg_dump.pl | 111 +++++++-
src/bin/pg_upgrade/dump.c | 6 +-
src/bin/pg_upgrade/option.c | 12 +
src/bin/pg_upgrade/pg_upgrade.h | 1 +
doc/src/sgml/ref/pg_dump.sgml | 80 ++++--
doc/src/sgml/ref/pg_dumpall.sgml | 42 ++-
doc/src/sgml/ref/pg_restore.sgml | 49 +++-
doc/src/sgml/ref/pgupgrade.sgml | 18 ++
src/tools/pgindent/typedefs.list | 1 +
19 files changed, 851 insertions(+), 72 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b29..3fa1474fad7 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,9 +110,12 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +163,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -179,8 +183,11 @@ typedef struct _dumpOptions
int column_inserts;
int if_exists;
int no_comments;
- int no_security_labels;
+ int no_data;
int no_publications;
+ int no_schema;
+ int no_security_labels;
+ int no_statistics;
int no_subscriptions;
int no_toast_compression;
int no_unlogged_table_data;
@@ -208,6 +215,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index 707a3fc844c..cfa1349b8a9 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -46,6 +46,12 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
+typedef enum entryType
+{
+ default_entry,
+ data_entry,
+ statistics_entry
+} entryType;
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -53,7 +59,7 @@ static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr,
DataDirSyncMethod sync_method);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
-static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
+static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type);
static char *sanitize_line(const char *str, bool want_hyphen);
static void _doSetFixedOutputState(ArchiveHandle *AH);
static void _doSetSessionAuth(ArchiveHandle *AH, const char *user);
@@ -149,6 +155,7 @@ InitDumpOptions(DumpOptions *opts)
opts->dumpSections = DUMP_UNSECTIONED;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
}
/*
@@ -169,9 +176,10 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->outputClean = ropt->dropSchema;
dopt->dumpData = ropt->dumpData;
dopt->dumpSchema = ropt->dumpSchema;
+ dopt->dumpSections = ropt->dumpSections;
+ dopt->dumpStatistics = ropt->dumpStatistics;
dopt->if_exists = ropt->if_exists;
dopt->column_inserts = ropt->column_inserts;
- dopt->dumpSections = ropt->dumpSections;
dopt->aclsSkip = ropt->aclsSkip;
dopt->outputSuperuser = ropt->superuser;
dopt->outputCreateDB = ropt->createDB;
@@ -186,6 +194,9 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
@@ -418,30 +429,30 @@ RestoreArchive(Archive *AHX)
}
/*
- * Work out if we have an implied data-only restore. This can happen if
- * the dump was data only or if the user has used a toc list to exclude
- * all of the schema data. All we do is look for schema entries - if none
- * are found then we unset the dumpSchema flag.
+ * Work out if we have an schema-less restore. This can happen if the dump
+ * was data-only or statistics-only or no-schema or if the user has used a
+ * toc list to exclude all of the schema data. All we do is look for
+ * schema entries - if none are found then we unset the dumpSchema flag.
*
* We could scan for wanted TABLE entries, but that is not the same as
* data-only. At this stage, it seems unnecessary (6-Mar-2001).
*/
if (ropt->dumpSchema)
{
- int impliedDataOnly = 1;
+ bool no_schema_found = true;
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if ((te->reqs & REQ_SCHEMA) != 0)
- { /* It's schema, and it's wanted */
- impliedDataOnly = 0;
+ {
+ no_schema_found = false;
break;
}
}
- if (impliedDataOnly)
+ if (no_schema_found)
{
ropt->dumpSchema = false;
- pg_log_info("implied data-only restore");
+ pg_log_info("implied no-schema restore");
}
}
@@ -739,7 +750,7 @@ RestoreArchive(Archive *AHX)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
continue; /* ignore if not to be dumped at all */
switch (_tocEntryRestorePass(te))
@@ -760,7 +771,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -770,7 +781,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_POST_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -869,7 +880,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
pg_log_info("creating %s \"%s\"",
te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
defnDumped = true;
if (strcmp(te->desc, "TABLE") == 0)
@@ -938,7 +949,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
*/
if (AH->PrintTocDataPtr != NULL)
{
- _printTocEntry(AH, te, true);
+ _printTocEntry(AH, te, data_entry);
if (strcmp(te->desc, "BLOBS") == 0 ||
strcmp(te->desc, "BLOB COMMENTS") == 0)
@@ -1036,15 +1047,21 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
{
/* If we haven't already dumped the defn part, do so now */
pg_log_info("executing %s %s", te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, default_entry);
}
}
+ /*
+ * If it has a statistics component that we want, then process that
+ */
+ if ((reqs & REQ_STATS) != 0)
+ _printTocEntry(AH, te, statistics_entry);
+
/*
* If we emitted anything for this TOC entry, that counts as one action
* against the transaction-size limit. Commit if it's time to.
*/
- if ((reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && ropt->txn_size > 0)
+ if ((reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && ropt->txn_size > 0)
{
if (++AH->txnCount >= ropt->txn_size)
{
@@ -1084,6 +1101,7 @@ NewRestoreOptions(void)
opts->compression_spec.level = 0;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
return opts;
}
@@ -1329,7 +1347,7 @@ PrintTOCSummary(Archive *AHX)
te->reqs = _tocEntryRequired(te, curSection, AH);
/* Now, should we print it? */
if (ropt->verbose ||
- (te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0)
+ (te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0)
{
char *sanitized_name;
char *sanitized_schema;
@@ -2582,7 +2600,7 @@ WriteToc(ArchiveHandle *AH)
tocCount = 0;
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) != 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) != 0)
tocCount++;
}
@@ -2592,7 +2610,7 @@ WriteToc(ArchiveHandle *AH)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) == 0)
continue;
WriteInt(AH, te->dumpId);
@@ -2918,6 +2936,14 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strcmp(te->desc, "SEARCHPATH") == 0)
return REQ_SPECIAL;
+ if (strcmp(te->desc, "STATISTICS DATA") == 0)
+ {
+ if (!ropt->dumpStatistics)
+ return 0;
+ else
+ res = REQ_STATS;
+ }
+
/*
* DATABASE and DATABASE PROPERTIES also have a special rule: they are
* restored in createDB mode, and not restored otherwise, independently of
@@ -2962,6 +2988,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's statistics and we don't want statistics, maybe ignore it */
+ if (!ropt->dumpStatistics && strcmp(te->desc, "STATISTICS DATA") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2991,6 +3021,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS DATA") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
@@ -3107,6 +3138,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
}
+
/*
* Determine whether the TOC entry contains schema and/or data components,
* and mask off inapplicable REQ bits. If it had a dataDumper, assume
@@ -3172,12 +3204,12 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strncmp(te->tag, "LARGE OBJECT", 12) == 0) ||
(strcmp(te->desc, "SECURITY LABEL") == 0 &&
strncmp(te->tag, "LARGE OBJECT", 12) == 0))))
- res = res & REQ_SCHEMA;
+ res = res & (REQ_SCHEMA | REQ_STATS);
}
/* Mask it if we don't want schema */
if (!ropt->dumpSchema)
- res = res & REQ_DATA;
+ res = res & (REQ_DATA | REQ_STATS);
return res;
}
@@ -3729,7 +3761,7 @@ _getObjectDescription(PQExpBuffer buf, const TocEntry *te)
* will remain at default, until the matching ACL TOC entry is restored.
*/
static void
-_printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
+_printTocEntry(ArchiveHandle *AH, TocEntry *te, entryType entry_type)
{
RestoreOptions *ropt = AH->public.ropt;
@@ -3753,10 +3785,17 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
char *sanitized_schema;
char *sanitized_owner;
- if (isData)
- pfx = "Data for ";
- else
- pfx = "";
+ switch (entry_type)
+ {
+ case data_entry:
+ pfx = "Data for ";
+ break;
+ case statistics_entry:
+ pfx = "Statistics for ";
+ break;
+ default:
+ pfx = "";
+ }
ahprintf(AH, "--\n");
if (AH->public.verbose)
@@ -4324,7 +4363,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
- if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
{
pg_log_info("skipping item %d %s %s",
next_work_item->dumpId,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index ce5ed1dd395..a2064f471ed 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -209,7 +209,8 @@ typedef enum
#define REQ_SCHEMA 0x01 /* want schema */
#define REQ_DATA 0x02 /* want data */
-#define REQ_SPECIAL 0x04 /* for special TOC entries */
+#define REQ_STATS 0x04
+#define REQ_SPECIAL 0x08 /* for special TOC entries */
struct _archiveHandle
{
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 240a1d41062..b2a841bb0ff 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -780,7 +780,7 @@ _PrepParallelRestore(ArchiveHandle *AH)
continue;
/* We may ignore items not due to be restored */
- if ((te->reqs & REQ_DATA) == 0)
+ if ((te->reqs & (REQ_DATA | REQ_STATS)) == 0)
continue;
/*
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 520e1338c28..2a2397e20c6 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -431,6 +431,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -467,6 +468,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -493,8 +495,11 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -540,7 +545,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -614,6 +619,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -785,6 +794,17 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -799,8 +819,9 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
@@ -1100,6 +1121,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1178,7 +1200,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1191,11 +1213,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1220,8 +1243,11 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
+ printf(_(" --no-schema do not dump schema\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6778,6 +6804,43 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+ info->postponed_def = false;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7203,6 +7266,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting || dopt->dumpStatistics)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7649,11 +7714,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7676,7 +7744,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7710,6 +7785,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10297,6 +10374,296 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * Decide which section to use based on the relkind of the parent object.
+ *
+ * NB: materialized views may be postponed from SECTION_PRE_DATA to
+ * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the
+ * matview stats will also be postponed to SECTION_POST_DATA. See
+ * repairMatViewBoundaryMultiLoop().
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+ switch (rsinfo->relkind)
+ {
+ case RELKIND_RELATION:
+ case RELKIND_PARTITIONED_TABLE:
+ case RELKIND_MATVIEW:
+ return SECTION_DATA;
+ case RELKIND_INDEX:
+ case RELKIND_PARTITIONED_INDEX:
+ return SECTION_POST_DATA;
+ default:
+ pg_fatal("cannot dump statistics for relation kind '%c'",
+ rsinfo->relkind);
+ }
+
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+ DumpId *deps = NULL;
+ int ndeps = 0;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ /* dependent on the relation definition, if doing schema */
+ if (fout->dopt->dumpSchema)
+ {
+ deps = dobj->dependencies;
+ ndeps = dobj->nDeps;
+ }
+
+ tag = createPQExpBuffer();
+ appendPQExpBuffer(tag, "%s %s", "STATISTICS DATA", fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = rsinfo->postponed_def ?
+ SECTION_POST_DATA : statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = deps,
+ .nDeps = ndeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10745,6 +11112,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -18974,6 +19344,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 7139c88a69a..410c162a85d 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -83,10 +83,13 @@ typedef enum
DO_PUBLICATION,
DO_PUBLICATION_REL,
DO_PUBLICATION_TABLE_IN_SCHEMA,
+ DO_REL_STATS,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
} DumpableObjectType;
+#define NUM_DUMPABLE_OBJECT_TYPES (DO_SUBSCRIPTION_REL + 1)
+
/*
* DumpComponents is a bitmask of the potentially dumpable components of
* a database object: its core definition, plus optional attributes such
@@ -110,6 +113,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -430,6 +434,13 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'm', 'i', etc */
+ bool postponed_def; /* stats must be postponed into post-data */
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index dc9a28924bd..201eeedde73 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -81,6 +81,7 @@ enum dbObjectTypePriorities
PRIO_TABLE_DATA,
PRIO_SEQUENCE_SET,
PRIO_LARGE_OBJECT_DATA,
+ PRIO_STATISTICS_DATA_DATA,
PRIO_POST_DATA_BOUNDARY, /* boundary! */
PRIO_CONSTRAINT,
PRIO_INDEX,
@@ -148,11 +149,12 @@ static const int dbObjectTypePriority[] =
[DO_PUBLICATION] = PRIO_PUBLICATION,
[DO_PUBLICATION_REL] = PRIO_PUBLICATION_REL,
[DO_PUBLICATION_TABLE_IN_SCHEMA] = PRIO_PUBLICATION_TABLE_IN_SCHEMA,
+ [DO_REL_STATS] = PRIO_STATISTICS_DATA_DATA,
[DO_SUBSCRIPTION] = PRIO_SUBSCRIPTION,
[DO_SUBSCRIPTION_REL] = PRIO_SUBSCRIPTION_REL,
};
-StaticAssertDecl(lengthof(dbObjectTypePriority) == (DO_SUBSCRIPTION_REL + 1),
+StaticAssertDecl(lengthof(dbObjectTypePriority) == NUM_DUMPABLE_OBJECT_TYPES,
"array length mismatch");
static DumpId preDataBoundId;
@@ -801,11 +803,21 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj,
{
/* remove boundary's dependency on object after it in loop */
removeObjectDependency(boundaryobj, nextobj->dumpId);
- /* if that object is a matview, mark it as postponed into post-data */
+ /*
+ * If that object is a matview or matview status, mark it as postponed into
+ * post-data.
+ */
if (nextobj->objType == DO_TABLE)
{
TableInfo *nextinfo = (TableInfo *) nextobj;
+ if (nextinfo->relkind == RELKIND_MATVIEW)
+ nextinfo->postponed_def = true;
+ }
+ else if (nextobj->objType == DO_REL_STATS)
+ {
+ RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj;
+
if (nextinfo->relkind == RELKIND_MATVIEW)
nextinfo->postponed_def = true;
}
@@ -1018,6 +1030,21 @@ repairDependencyLoop(DumpableObject **loop,
{
DumpableObject *nextobj;
+ nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
+ repairMatViewBoundaryMultiLoop(loop[j], nextobj);
+ return;
+ }
+ }
+ }
+ else if (loop[i]->objType == DO_REL_STATS &&
+ ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW)
+ {
+ for (j = 0; j < nLoop; j++)
+ {
+ if (loop[j]->objType == DO_POST_DATA_BOUNDARY)
+ {
+ DumpableObject *nextobj;
+
nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
repairMatViewBoundaryMultiLoop(loop[j], nextobj);
return;
@@ -1500,6 +1527,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 396f79781c5..7effb704905 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,7 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -172,6 +173,7 @@ main(int argc, char *argv[])
{"no-role-passwords", no_argument, &no_role_passwords, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -451,6 +453,8 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -666,6 +670,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index c602272d7db..02af89bae15 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,9 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool data_only = false;
+ bool schema_only = false;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -71,12 +74,13 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_comments = 0;
+ static int no_data = 0;
static int no_publications = 0;
+ static int no_schema = 0;
static int no_security_labels = 0;
+ static int no_statistics = 0;
static int no_subscriptions = 0;
static int strict_names = 0;
- bool data_only = false;
- bool schema_only = false;
struct option cmdopts[] = {
{"clean", 0, NULL, 'c'},
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -124,9 +129,12 @@ main(int argc, char **argv)
{"transaction-size", required_argument, NULL, 5},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
@@ -271,6 +279,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +355,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +378,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpData = data_only || (!no_data && !schema_only && !statistics_only);
+ opts->dumpSchema = schema_only || (!no_schema && !data_only && !statistics_only);
+ opts->dumpStatistics = statistics_only || (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +392,8 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
@@ -482,6 +501,7 @@ usage(const char *progname)
printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n"));
printf(_(" -T, --trigger=NAME restore named trigger\n"));
printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only restore only the statistics, not schema or data\n"));
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
@@ -489,10 +509,13 @@ usage(const char *progname)
" in FILENAME\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comment commands\n"));
+ printf(_(" --no-data do not restore data\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 214240f1ae5..f29da06ed28 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index bc5d9222a20..e1545e797a7 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -65,7 +65,7 @@ my %pgdump_runs = (
'--format' => 'custom',
'--file' => "$tempdir/binary_upgrade.dump",
'--no-password',
- '--schema-only',
+ '--no-data',
'--binary-upgrade',
'--dbname' => 'postgres', # alternative way to specify database
],
@@ -710,6 +710,39 @@ my %pgdump_runs = (
'--no-large-objects',
'postgres',
],
+ },
+ no_statistics => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_statistics.sql",
+ '--no-statistics',
+ 'postgres',
+ ],
+ },
+ no_data_no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_data_no_schema.sql",
+ '--no-data',
+ '--no-schema',
+ 'postgres',
+ ],
+ },
+ statistics_only => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/statistics_only.sql",
+ '--statistics-only',
+ 'postgres',
+ ],
+ },
+ no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_schema.sql",
+ '--no-schema',
+ 'postgres',
+ ],
},);
###############################################################
@@ -776,6 +809,7 @@ my %full_runs = (
no_large_objects => 1,
no_owner => 1,
no_privs => 1,
+ no_statistics => 1,
no_table_access_method => 1,
pg_dumpall_dbprivs => 1,
pg_dumpall_exclude => 1,
@@ -977,6 +1011,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1390,6 +1425,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1411,6 +1447,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1432,6 +1469,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1598,6 +1636,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1751,6 +1790,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_test_table => 1,
section_data => 1,
},
@@ -1778,6 +1818,7 @@ my %tests = (
data_only => 1,
exclude_test_table => 1,
exclude_test_table_data => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1798,7 +1839,10 @@ my %tests = (
\QCOPY dump_test.fk_reference_test_table (col1) FROM stdin;\E
\n(?:\d\n){5}\\\.\n
/xms,
- like => { data_only => 1, },
+ like => {
+ data_only => 1,
+ no_schema => 1,
+ },
},
'COPY test_second_table' => {
@@ -1814,6 +1858,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1836,6 +1881,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1859,6 +1905,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1881,6 +1928,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1903,6 +1951,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -3299,6 +3348,7 @@ my %tests = (
like => {
%full_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
only_dump_test_schema => 1,
test_schema_plus_large_objects => 1,
@@ -3469,6 +3519,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_measurement => 1,
section_data => 1,
only_dump_test_schema => 1,
@@ -4353,6 +4404,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
binary_upgrade => 1,
@@ -4653,6 +4705,61 @@ my %tests = (
},
},
+ #
+ # TABLE and MATVIEW stats will end up in SECTION_DATA.
+ # INDEX stats (expression columns only) will end up in SECTION_POST_DATA.
+ #
+ 'statistics_import' => {
+ create_sql => '
+ CREATE TABLE dump_test.has_stats
+ AS SELECT g.g AS x, g.g / 2 AS y FROM generate_series(1,100) AS g(g);
+ CREATE MATERIALIZED VIEW dump_test.has_stats_mv AS SELECT * FROM dump_test.has_stats;
+ CREATE INDEX dup_test_post_data_ix ON dump_test.has_stats((x - 1));
+ ANALYZE dump_test.has_stats, dump_test.has_stats_mv;',
+ regexp => qr/pg_catalog.pg_restore_attribute_stats/,
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ exclude_dump_test_schema => 1,
+ no_statistics => 1,
+ only_dump_measurement => 1,
+ schema_only => 1,
+ },
+ },
+
+ #
+ # While attribute stats (aka pg_statistic stats) only appear for tables
+ # that have been analyzed, all tables will have relation stats because
+ # those come from pg_class.
+ #
+ 'relstats_on_unanalyzed_tables' => {
+ regexp => qr/pg_catalog.pg_restore_relation_stats/,
+
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ only_dump_test_table => 1,
+ role => 1,
+ role_parallel => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ no_statistics => 1,
+ schema_only => 1,
+ },
+ },
+
# CREATE TABLE with partitioned table and various AMs. One
# partition uses the same default as the parent, and a second
# uses its own AM.
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8ce0fa3020e..a29cd2cca98 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,10 +21,11 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
"--binary-upgrade %s --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
GLOBALS_DUMP_FILE);
check_ok();
@@ -52,10 +53,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 108eb7a1ba4..3b6c7ec994e 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -306,7 +316,9 @@ usage(void)
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
+ printf(_(" --no-statistics do not import statistics from old cluster\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statistics import statistics from old cluster (default)\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 0cdd675e4f1..3fe111fbde5 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 24fcc76d72c..0d5e7d5a2d7 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,13 +141,11 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
- switch is therefore only useful to add large objects to dumps
- where a specific schema or table has been requested. Note that
- large objects are considered data and therefore will be included when
- <option>--data-only</option> is used, but not
- when <option>--schema-only</option> is.
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or <option>--statistics-only</option>, or
+ <option>--no-data</option> is specified. The <option>-b</option>
+ switch is therefore only useful to add large objects to dumps where a
+ specific schema or table has been requested.
</para>
</listitem>
</varlistentry>
@@ -516,10 +514,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +651,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -741,7 +752,8 @@ PostgreSQL documentation
<term><option>--disable-triggers</option></term>
<listitem>
<para>
- This option is relevant only when creating a data-only dump.
+ This option is relevant only when creating a dump that includes data
+ but does not include schema.
It instructs <application>pg_dump</application> to include commands
to temporarily disable triggers on the target tables while
the data is restored. Use this if you have referential
@@ -824,16 +836,17 @@ PostgreSQL documentation
<term><option>--exclude-table-data=<replaceable class="parameter">pattern</replaceable></option></term>
<listitem>
<para>
- Do not dump data for any tables matching <replaceable
+ Do not dump data or statistics for any tables matching <replaceable
class="parameter">pattern</replaceable>. The pattern is
interpreted according to the same rules as for <option>-t</option>.
<option>--exclude-table-data</option> can be given more than once to
- exclude tables matching any of several patterns. This option is
- useful when you need the definition of a particular table even
- though you do not need the data in it.
+ exclude tables matching any of several patterns. This option is useful
+ when you need the definition of a particular table even though you do
+ not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1080,6 +1093,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -1098,6 +1120,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
@@ -1236,9 +1276,11 @@ PostgreSQL documentation
</para>
<para>
The data section contains actual table data, large-object
- contents, and sequence values.
+ contents, statitistics for tables and materialized views and
+ sequence values.
Post-data items include definitions of indexes, triggers, rules,
- and constraints other than validated check constraints.
+ statistics for indexes, and constraints other than validated check
+ constraints.
Pre-data items include all other data definition items.
</para>
</listitem>
@@ -1581,7 +1623,7 @@ CREATE DATABASE foo WITH TEMPLATE template0;
</para>
<para>
- When a data-only dump is chosen and the option <option>--disable-triggers</option>
+ When a dump without schema is chosen and the option <option>--disable-triggers</option>
is used, <application>pg_dump</application> emits commands
to disable triggers on user tables before inserting the data,
and then commands to re-enable them after the data has been
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index 39d93c2c0e3..15fb40e7be9 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -81,7 +81,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
</para>
</listitem>
</varlistentry>
@@ -265,6 +265,17 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--binary-upgrade</option></term>
<listitem>
@@ -307,7 +318,7 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
<term><option>--disable-triggers</option></term>
<listitem>
<para>
- This option is relevant only when creating a data-only dump.
+ This option is relevant only when creating a dump with data and without schema.
It instructs <application>pg_dumpall</application> to include commands
to temporarily disable triggers on the target tables while
the data is restored. Use this if you have referential
@@ -422,6 +433,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -447,6 +467,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -456,6 +485,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719e..86e4264714d 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema (data definitions) or data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -617,7 +628,7 @@ PostgreSQL documentation
<term><option>--disable-triggers</option></term>
<listitem>
<para>
- This option is relevant only when performing a data-only restore.
+ This option is relevant only when performing a restore without schema.
It instructs <application>pg_restore</application> to execute commands
to temporarily disable triggers on the target tables while
the data is restored. Use this if you have referential
@@ -681,6 +692,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore data, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-data-for-failed-tables</option></term>
<listitem>
@@ -713,6 +734,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema (data definitions), even if
+ the archive contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -723,6 +754,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml
index 4777381dac2..64a1ebd613b 100644
--- a/doc/src/sgml/ref/pgupgrade.sgml
+++ b/doc/src/sgml/ref/pgupgrade.sgml
@@ -145,6 +145,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-statistics</option></term>
+ <listitem>
+ <para>
+ Restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-o</option> <replaceable class="parameter">options</replaceable></term>
<term><option>--old-options</option> <replaceable class="parameter">options</replaceable></term>
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9a3bee93dec..f3351342a24 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2400,6 +2400,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelStatsInfo
RelToCheck
RelToCluster
RelabelType
--
2.48.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-10 23:36 ` Jeff Davis <[email protected]>
1 sibling, 0 replies; 81+ messages in thread
From: Jeff Davis @ 2025-02-10 23:36 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; +Cc: jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Sun, 2025-02-09 at 22:00 -0500, Corey Huinker wrote:
>
> 0001 - I've added pg_locks tests for a regular index and a
> partitioned index.
Committed 0001, the fix for importing stats.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-11 01:21 ` Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2025-02-11 01:21 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; +Cc: jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Sun, 2025-02-09 at 22:00 -0500, Corey Huinker wrote:
>
> 0002 - I've done some documentation rewording, mostly wording changes
> where behaviors surrounding data-only dumps are actually meant for
> any dump that has all schema excluded.
Comments on v45-0002:
* Why is generate_old_dump() passing optionally passing --no-statistics
to pg_dumpall along with --globals-only? If --globals-only is
specified, no stats are dumped anyway, right?
* The tag is still wrong: it is "STATISTICS DATA mytable" when it
should just be "mytable".
* What's the logic behind the pg_dumpall options? The docs say
it should support the new pg_dump options, but they don't seem to work.
* The enum entryType casing is unconventional. How about a type name of
TocEntryType and values like STATS_TOC_ENTRY.
* The pg_dump test suite time has increased by ~50%. If some tests are
superfluous, please remove them.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-11 19:02 ` Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-02-11 19:02 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
>
> Comments on v45-0002:
>
Assuming you meant 47
>
> * Why is generate_old_dump() passing optionally passing --no-statistics
> to pg_dumpall along with --globals-only? If --globals-only is
> specified, no stats are dumped anyway, right?
>
Removed.
>
> * The tag is still wrong: it is "STATISTICS DATA mytable" when it
> should just be "mytable".
>
Fixed.
>
> * What's the logic behind the pg_dumpall options? The docs say
> it should support the new pg_dump options, but they don't seem to work.
>
Fixed.
>
> * The enum entryType casing is unconventional. How about a type name of
> TocEntryType and values like STATS_TOC_ENTRY.
>
Conventionified.
>
> * The pg_dump test suite time has increased by ~50%. If some tests are
> superfluous, please remove them.
The nature of the TAP tests is that every new check N must be run against
every existing dump run M adding one N check gets you M scans, and adding a
dump type requires that it be scanned N times, and we increased both N and
M. If there were a way to only do stats-related tests against a subset of
the dumps, then yes, we could trim it down, but as it is I think it's a
limitation of the testing structure. But aside from creating a whole extra
XYZ_pg_dump.pl file, I don't think there's a way to do that.
The previous 0001 is now committed (thanks!) so only one remains.
Attachments:
[text/x-patch] v48-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch (70.9K, ../../CADkLM=cALMHZkXU+xZw9-WeR4XSXzDYj_-xERHg8haYq1zbjhg@mail.gmail.com/3-v48-0001-Enable-dumping-of-table-index-stats-in-pg_dump.patch)
download | inline diff:
From afb3d0fd81fb5395ff71ac7b05131babf5d61877 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Sat, 16 Mar 2024 17:21:10 -0400
Subject: [PATCH v48] Enable dumping of table/index stats in pg_dump.
For each table/matview/index dumped, it will generate a statement that
calls pg_set_relation_stats(), and it will generate a series of
statements that call pg_set_attribute_stats(), one per attribute that
has a corresponding row in pg_statistic. These statements will restore
the statistics of the current system onto the destination system.
Adds the command-line options -X / --statistics-only, which are mutually
exclusive to --schema-only and --data-only.
Statistics are not dumped when --schema-only is specified, except during
a binary upgrade.
As is the pattern with pg_dump options, statistics can be disabled using
--no-statistics.
Table statistics are dumped in the data section. This is true even if
dumping stats in a binary upgrade. Index statistics are dumped in the
post-data section.
Add --no-data option. This option is useful for situations where
someone wishes to test query plans from a production database without
copying production data.
This also makes the corresponding change to the simulated pg_upgrade in
the TAP tests for pg_dump.
Add --no-schema option to pg_dump, etc. Previously, users could use
--data-only when they wanted to suppress schema from a dump. However,
that no longer makes sense now that the data/schema binary has become
the data/schema/statistics trinary.
---
src/bin/pg_dump/pg_backup.h | 10 +-
src/bin/pg_dump/pg_backup_archiver.c | 97 +++++--
src/bin/pg_dump/pg_backup_archiver.h | 3 +-
src/bin/pg_dump/pg_backup_directory.c | 2 +-
src/bin/pg_dump/pg_dump.c | 390 +++++++++++++++++++++++++-
src/bin/pg_dump/pg_dump.h | 11 +
src/bin/pg_dump/pg_dump_sort.c | 36 ++-
src/bin/pg_dump/pg_dumpall.c | 18 ++
src/bin/pg_dump/pg_restore.c | 31 +-
src/bin/pg_dump/t/001_basic.pl | 18 ++
src/bin/pg_dump/t/002_pg_dump.pl | 111 +++++++-
src/bin/pg_upgrade/dump.c | 7 +-
src/bin/pg_upgrade/option.c | 12 +
src/bin/pg_upgrade/pg_upgrade.h | 1 +
doc/src/sgml/ref/pg_dump.sgml | 80 ++++--
doc/src/sgml/ref/pg_dumpall.sgml | 42 ++-
doc/src/sgml/ref/pg_restore.sgml | 49 +++-
doc/src/sgml/ref/pgupgrade.sgml | 18 ++
src/tools/pgindent/typedefs.list | 1 +
19 files changed, 864 insertions(+), 73 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index f0f19bb0b29..3fa1474fad7 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -110,9 +110,12 @@ typedef struct _restoreOptions
int column_inserts;
int if_exists;
int no_comments; /* Skip comments */
+ int no_data; /* Skip data */
int no_publications; /* Skip publication entries */
+ int no_schema; /* Skip schema generation */
int no_security_labels; /* Skip security label entries */
int no_subscriptions; /* Skip subscription entries */
+ int no_statistics; /* Skip statistics import */
int strict_names;
const char *filename;
@@ -160,6 +163,7 @@ typedef struct _restoreOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} RestoreOptions;
typedef struct _dumpOptions
@@ -179,8 +183,11 @@ typedef struct _dumpOptions
int column_inserts;
int if_exists;
int no_comments;
- int no_security_labels;
+ int no_data;
int no_publications;
+ int no_schema;
+ int no_security_labels;
+ int no_statistics;
int no_subscriptions;
int no_toast_compression;
int no_unlogged_table_data;
@@ -208,6 +215,7 @@ typedef struct _dumpOptions
/* flags derived from the user-settable flags */
bool dumpSchema;
bool dumpData;
+ bool dumpStatistics;
} DumpOptions;
/*
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index b9d7ab98c3e..6c897311437 100644
--- a/src/bin/pg_dump/pg_backup_archiver.c
+++ b/src/bin/pg_dump/pg_backup_archiver.c
@@ -46,6 +46,12 @@
#define TEXT_DUMP_HEADER "--\n-- PostgreSQL database dump\n--\n\n"
#define TEXT_DUMPALL_HEADER "--\n-- PostgreSQL database cluster dump\n--\n\n"
+typedef enum TocEntryType
+{
+ DEFAULT_TOC_ENTRY,
+ DATA_TOC_ENTRY,
+ STATS_TOC_ENTRY
+} TocEntryType;
static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
const pg_compress_specification compression_spec,
@@ -53,7 +59,7 @@ static ArchiveHandle *_allocAH(const char *FileSpec, const ArchiveFormat fmt,
SetupWorkerPtrType setupWorkerPtr,
DataDirSyncMethod sync_method);
static void _getObjectDescription(PQExpBuffer buf, const TocEntry *te);
-static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData);
+static void _printTocEntry(ArchiveHandle *AH, TocEntry *te, TocEntryType entry_type);
static char *sanitize_line(const char *str, bool want_hyphen);
static void _doSetFixedOutputState(ArchiveHandle *AH);
static void _doSetSessionAuth(ArchiveHandle *AH, const char *user);
@@ -149,6 +155,7 @@ InitDumpOptions(DumpOptions *opts)
opts->dumpSections = DUMP_UNSECTIONED;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
}
/*
@@ -169,9 +176,10 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->outputClean = ropt->dropSchema;
dopt->dumpData = ropt->dumpData;
dopt->dumpSchema = ropt->dumpSchema;
+ dopt->dumpSections = ropt->dumpSections;
+ dopt->dumpStatistics = ropt->dumpStatistics;
dopt->if_exists = ropt->if_exists;
dopt->column_inserts = ropt->column_inserts;
- dopt->dumpSections = ropt->dumpSections;
dopt->aclsSkip = ropt->aclsSkip;
dopt->outputSuperuser = ropt->superuser;
dopt->outputCreateDB = ropt->createDB;
@@ -186,6 +194,9 @@ dumpOptionsFromRestoreOptions(RestoreOptions *ropt)
dopt->no_publications = ropt->no_publications;
dopt->no_security_labels = ropt->no_security_labels;
dopt->no_subscriptions = ropt->no_subscriptions;
+ dopt->no_data = ropt->no_data;
+ dopt->no_schema = ropt->no_schema;
+ dopt->no_statistics = ropt->no_statistics;
dopt->lockWaitTimeout = ropt->lockWaitTimeout;
dopt->include_everything = ropt->include_everything;
dopt->enable_row_security = ropt->enable_row_security;
@@ -418,30 +429,30 @@ RestoreArchive(Archive *AHX)
}
/*
- * Work out if we have an implied data-only restore. This can happen if
- * the dump was data only or if the user has used a toc list to exclude
- * all of the schema data. All we do is look for schema entries - if none
- * are found then we unset the dumpSchema flag.
+ * Work out if we have an schema-less restore. This can happen if the dump
+ * was data-only or statistics-only or no-schema or if the user has used a
+ * toc list to exclude all of the schema data. All we do is look for
+ * schema entries - if none are found then we unset the dumpSchema flag.
*
* We could scan for wanted TABLE entries, but that is not the same as
* data-only. At this stage, it seems unnecessary (6-Mar-2001).
*/
if (ropt->dumpSchema)
{
- int impliedDataOnly = 1;
+ bool no_schema_found = true;
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
if ((te->reqs & REQ_SCHEMA) != 0)
- { /* It's schema, and it's wanted */
- impliedDataOnly = 0;
+ {
+ no_schema_found = false;
break;
}
}
- if (impliedDataOnly)
+ if (no_schema_found)
{
ropt->dumpSchema = false;
- pg_log_info("implied data-only restore");
+ pg_log_info("implied no-schema restore");
}
}
@@ -739,7 +750,7 @@ RestoreArchive(Archive *AHX)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
continue; /* ignore if not to be dumped at all */
switch (_tocEntryRestorePass(te))
@@ -760,7 +771,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -770,7 +781,7 @@ RestoreArchive(Archive *AHX)
{
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0 &&
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 &&
_tocEntryRestorePass(te) == RESTORE_PASS_POST_ACL)
(void) restore_toc_entry(AH, te, false);
}
@@ -869,7 +880,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
pg_log_info("creating %s \"%s\"",
te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, DEFAULT_TOC_ENTRY);
defnDumped = true;
if (strcmp(te->desc, "TABLE") == 0)
@@ -938,7 +949,7 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
*/
if (AH->PrintTocDataPtr != NULL)
{
- _printTocEntry(AH, te, true);
+ _printTocEntry(AH, te, DATA_TOC_ENTRY);
if (strcmp(te->desc, "BLOBS") == 0 ||
strcmp(te->desc, "BLOB COMMENTS") == 0)
@@ -1036,15 +1047,21 @@ restore_toc_entry(ArchiveHandle *AH, TocEntry *te, bool is_parallel)
{
/* If we haven't already dumped the defn part, do so now */
pg_log_info("executing %s %s", te->desc, te->tag);
- _printTocEntry(AH, te, false);
+ _printTocEntry(AH, te, DEFAULT_TOC_ENTRY);
}
}
+ /*
+ * If it has a statistics component that we want, then process that
+ */
+ if ((reqs & REQ_STATS) != 0)
+ _printTocEntry(AH, te, STATS_TOC_ENTRY);
+
/*
* If we emitted anything for this TOC entry, that counts as one action
* against the transaction-size limit. Commit if it's time to.
*/
- if ((reqs & (REQ_SCHEMA | REQ_DATA)) != 0 && ropt->txn_size > 0)
+ if ((reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0 && ropt->txn_size > 0)
{
if (++AH->txnCount >= ropt->txn_size)
{
@@ -1084,6 +1101,7 @@ NewRestoreOptions(void)
opts->compression_spec.level = 0;
opts->dumpSchema = true;
opts->dumpData = true;
+ opts->dumpStatistics = true;
return opts;
}
@@ -1329,7 +1347,7 @@ PrintTOCSummary(Archive *AHX)
te->reqs = _tocEntryRequired(te, curSection, AH);
/* Now, should we print it? */
if (ropt->verbose ||
- (te->reqs & (REQ_SCHEMA | REQ_DATA)) != 0)
+ (te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) != 0)
{
char *sanitized_name;
char *sanitized_schema;
@@ -2582,7 +2600,7 @@ WriteToc(ArchiveHandle *AH)
tocCount = 0;
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) != 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) != 0)
tocCount++;
}
@@ -2592,7 +2610,7 @@ WriteToc(ArchiveHandle *AH)
for (te = AH->toc->next; te != AH->toc; te = te->next)
{
- if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_SPECIAL)) == 0)
+ if ((te->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS | REQ_SPECIAL)) == 0)
continue;
WriteInt(AH, te->dumpId);
@@ -2919,6 +2937,14 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strcmp(te->desc, "SEARCHPATH") == 0)
return REQ_SPECIAL;
+ if (strcmp(te->desc, "STATISTICS DATA") == 0)
+ {
+ if (!ropt->dumpStatistics)
+ return 0;
+ else
+ res = REQ_STATS;
+ }
+
/*
* DATABASE and DATABASE PROPERTIES also have a special rule: they are
* restored in createDB mode, and not restored otherwise, independently of
@@ -2963,6 +2989,10 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
if (ropt->no_subscriptions && strcmp(te->desc, "SUBSCRIPTION") == 0)
return 0;
+ /* If it's statistics and we don't want statistics, maybe ignore it */
+ if (!ropt->dumpStatistics && strcmp(te->desc, "STATISTICS DATA") == 0)
+ return 0;
+
/* Ignore it if section is not to be dumped/restored */
switch (curSection)
{
@@ -2992,6 +3022,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
*/
if (strcmp(te->desc, "ACL") == 0 ||
strcmp(te->desc, "COMMENT") == 0 ||
+ strcmp(te->desc, "STATISTICS DATA") == 0 ||
strcmp(te->desc, "SECURITY LABEL") == 0)
{
/* Database properties react to createDB, not selectivity options. */
@@ -3108,6 +3139,7 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
}
}
+
/*
* Determine whether the TOC entry contains schema and/or data components,
* and mask off inapplicable REQ bits. If it had a dataDumper, assume
@@ -3173,12 +3205,12 @@ _tocEntryRequired(TocEntry *te, teSection curSection, ArchiveHandle *AH)
strncmp(te->tag, "LARGE OBJECT", 12) == 0) ||
(strcmp(te->desc, "SECURITY LABEL") == 0 &&
strncmp(te->tag, "LARGE OBJECT", 12) == 0))))
- res = res & REQ_SCHEMA;
+ res = res & (REQ_SCHEMA | REQ_STATS);
}
/* Mask it if we don't want schema */
if (!ropt->dumpSchema)
- res = res & REQ_DATA;
+ res = res & (REQ_DATA | REQ_STATS);
return res;
}
@@ -3730,7 +3762,7 @@ _getObjectDescription(PQExpBuffer buf, const TocEntry *te)
* will remain at default, until the matching ACL TOC entry is restored.
*/
static void
-_printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
+_printTocEntry(ArchiveHandle *AH, TocEntry *te, TocEntryType entry_type)
{
RestoreOptions *ropt = AH->public.ropt;
@@ -3754,10 +3786,17 @@ _printTocEntry(ArchiveHandle *AH, TocEntry *te, bool isData)
char *sanitized_schema;
char *sanitized_owner;
- if (isData)
- pfx = "Data for ";
- else
- pfx = "";
+ switch (entry_type)
+ {
+ case DATA_TOC_ENTRY:
+ pfx = "Data for ";
+ break;
+ case STATS_TOC_ENTRY:
+ pfx = "Statistics for ";
+ break;
+ default:
+ pfx = "";
+ }
ahprintf(AH, "--\n");
if (AH->public.verbose)
@@ -4325,7 +4364,7 @@ restore_toc_entries_parallel(ArchiveHandle *AH, ParallelState *pstate,
if (next_work_item != NULL)
{
/* If not to be restored, don't waste time launching a worker */
- if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA)) == 0)
+ if ((next_work_item->reqs & (REQ_SCHEMA | REQ_DATA | REQ_STATS)) == 0)
{
pg_log_info("skipping item %d %s %s",
next_work_item->dumpId,
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index ce5ed1dd395..a2064f471ed 100644
--- a/src/bin/pg_dump/pg_backup_archiver.h
+++ b/src/bin/pg_dump/pg_backup_archiver.h
@@ -209,7 +209,8 @@ typedef enum
#define REQ_SCHEMA 0x01 /* want schema */
#define REQ_DATA 0x02 /* want data */
-#define REQ_SPECIAL 0x04 /* for special TOC entries */
+#define REQ_STATS 0x04
+#define REQ_SPECIAL 0x08 /* for special TOC entries */
struct _archiveHandle
{
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index 240a1d41062..b2a841bb0ff 100644
--- a/src/bin/pg_dump/pg_backup_directory.c
+++ b/src/bin/pg_dump/pg_backup_directory.c
@@ -780,7 +780,7 @@ _PrepParallelRestore(ArchiveHandle *AH)
continue;
/* We may ignore items not due to be restored */
- if ((te->reqs & REQ_DATA) == 0)
+ if ((te->reqs & (REQ_DATA | REQ_STATS)) == 0)
continue;
/*
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index ca15b40939c..11048fc98ba 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -431,6 +431,7 @@ main(int argc, char **argv)
DataDirSyncMethod sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
bool data_only = false;
bool schema_only = false;
+ bool statistics_only = false;
static DumpOptions dopt;
@@ -467,6 +468,7 @@ main(int argc, char **argv)
{"encoding", required_argument, NULL, 'E'},
{"help", no_argument, NULL, '?'},
{"version", no_argument, NULL, 'V'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -493,8 +495,11 @@ main(int argc, char **argv)
{"strict-names", no_argument, &strict_names, 1},
{"use-set-session-authorization", no_argument, &dopt.use_setsessauth, 1},
{"no-comments", no_argument, &dopt.no_comments, 1},
+ {"no-data", no_argument, &dopt.no_data, 1},
{"no-publications", no_argument, &dopt.no_publications, 1},
+ {"no-schema", no_argument, &dopt.no_schema, 1},
{"no-security-labels", no_argument, &dopt.no_security_labels, 1},
+ {"no-statistics", no_argument, &dopt.no_statistics, 1},
{"no-subscriptions", no_argument, &dopt.no_subscriptions, 1},
{"no-toast-compression", no_argument, &dopt.no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1},
@@ -540,7 +545,7 @@ main(int argc, char **argv)
InitDumpOptions(&dopt);
- while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxZ:",
+ while ((c = getopt_long(argc, argv, "abBcCd:e:E:f:F:h:j:n:N:Op:RsS:t:T:U:vwWxXZ:",
long_options, &optindex)) != -1)
{
switch (c)
@@ -614,6 +619,10 @@ main(int argc, char **argv)
dopt.cparams.pgport = pg_strdup(optarg);
break;
+ case 'X': /* Dump statistics only */
+ statistics_only = true;
+ break;
+
case 'R':
/* no-op, still accepted for backwards compatibility */
break;
@@ -785,6 +794,17 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+
+ if (data_only && dopt.no_data)
+ pg_fatal("options -a/--data-only and --no-data cannot be used together");
+ if (schema_only && dopt.no_schema)
+ pg_fatal("options -s/--schema-only and --no-schema cannot be used together");
+ if (statistics_only && dopt.no_statistics)
+ pg_fatal("options -X/--statistics-only and --no-statistics cannot be used together");
if (schema_only && foreign_servers_include_patterns.head != NULL)
pg_fatal("options -s/--schema-only and --include-foreign-data cannot be used together");
@@ -799,8 +819,9 @@ main(int argc, char **argv)
pg_fatal("option --if-exists requires option -c/--clean");
/* set derivative flags */
- dopt.dumpSchema = (!data_only);
- dopt.dumpData = (!schema_only);
+ dopt.dumpData = data_only || (!schema_only && !statistics_only && !dopt.no_data);
+ dopt.dumpSchema = schema_only || (!data_only && !statistics_only && !dopt.no_schema);
+ dopt.dumpStatistics = statistics_only || (!data_only && !schema_only && !dopt.no_statistics);
/*
* --inserts are already implied above if --column-inserts or
@@ -1100,6 +1121,7 @@ main(int argc, char **argv)
ropt->dropSchema = dopt.outputClean;
ropt->dumpData = dopt.dumpData;
ropt->dumpSchema = dopt.dumpSchema;
+ ropt->dumpStatistics = dopt.dumpStatistics;
ropt->if_exists = dopt.if_exists;
ropt->column_inserts = dopt.column_inserts;
ropt->dumpSections = dopt.dumpSections;
@@ -1178,7 +1200,7 @@ help(const char *progname)
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nOptions controlling the output content:\n"));
- printf(_(" -a, --data-only dump only the data, not the schema\n"));
+ printf(_(" -a, --data-only dump only the data, not the schema or statistics\n"));
printf(_(" -b, --large-objects include large objects in dump\n"));
printf(_(" --blobs (same as --large-objects, deprecated)\n"));
printf(_(" -B, --no-large-objects exclude large objects in dump\n"));
@@ -1191,11 +1213,12 @@ help(const char *progname)
printf(_(" -N, --exclude-schema=PATTERN do NOT dump the specified schema(s)\n"));
printf(_(" -O, --no-owner skip restoration of object ownership in\n"
" plain-text format\n"));
- printf(_(" -s, --schema-only dump only the schema, no data\n"));
+ printf(_(" -s, --schema-only dump only the schema, no data or statistics\n"));
printf(_(" -S, --superuser=NAME superuser user name to use in plain-text format\n"));
printf(_(" -t, --table=PATTERN dump only the specified table(s)\n"));
printf(_(" -T, --exclude-table=PATTERN do NOT dump the specified table(s)\n"));
printf(_(" -x, --no-privileges do not dump privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only dump only the statistics, not schema or data\n"));
printf(_(" --binary-upgrade for use by upgrade utilities only\n"));
printf(_(" --column-inserts dump data as INSERT commands with column names\n"));
printf(_(" --disable-dollar-quoting disable dollar quoting, use SQL standard quoting\n"));
@@ -1220,8 +1243,11 @@ help(const char *progname)
printf(_(" --inserts dump data as INSERT commands, rather than COPY\n"));
printf(_(" --load-via-partition-root load partitions via the root table\n"));
printf(_(" --no-comments do not dump comment commands\n"));
+ printf(_(" --no-data do not dump data\n"));
printf(_(" --no-publications do not dump publications\n"));
+ printf(_(" --no-schema do not dump schema\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
printf(_(" --no-tablespaces do not dump tablespace assignments\n"));
@@ -6779,6 +6805,43 @@ getFuncs(Archive *fout)
destroyPQExpBuffer(query);
}
+/*
+ * getRelationStatistics
+ * register the statistics object as a dependent of the relation.
+ *
+ */
+static RelStatsInfo *
+getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+{
+ if ((relkind == RELKIND_RELATION) ||
+ (relkind == RELKIND_PARTITIONED_TABLE) ||
+ (relkind == RELKIND_INDEX) ||
+ (relkind == RELKIND_PARTITIONED_INDEX) ||
+ (relkind == RELKIND_MATVIEW))
+ {
+ RelStatsInfo *info = pg_malloc0(sizeof(RelStatsInfo));
+ DumpableObject *dobj = &info->dobj;
+
+ dobj->objType = DO_REL_STATS;
+ dobj->catId.tableoid = 0;
+ dobj->catId.oid = 0;
+ AssignDumpId(dobj);
+ dobj->dependencies = (DumpId *) pg_malloc(sizeof(DumpId));
+ dobj->dependencies[0] = rel->dumpId;
+ dobj->nDeps = 1;
+ dobj->allocDeps = 1;
+ dobj->components |= DUMP_COMPONENT_STATISTICS;
+ dobj->dump = rel->dump;
+ dobj->name = pg_strdup(rel->name);
+ dobj->namespace = rel->namespace;
+ info->relkind = relkind;
+ info->postponed_def = false;
+
+ return info;
+ }
+ return NULL;
+}
+
/*
* getTables
* read all the tables (no indexes) in the system catalogs,
@@ -7204,6 +7267,8 @@ getTables(Archive *fout, int *numTables)
}
}
}
+ if (tblinfo[i].interesting || dopt->dumpStatistics)
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
}
if (query->len != 0)
@@ -7650,11 +7715,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
for (int c = 0; c < numinds; c++, j++)
{
char contype;
+ char indexkind;
+ RelStatsInfo *relstats;
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
indxinfo[j].dobj.catId.oid = atooid(PQgetvalue(res, j, i_oid));
AssignDumpId(&indxinfo[j].dobj);
+ indxinfo[j].dobj.components |= DUMP_COMPONENT_STATISTICS;
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
@@ -7677,7 +7745,14 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
{
NULL, NULL
};
+
+ if (indxinfo[j].parentidx == 0)
+ indexkind = RELKIND_INDEX;
+ else
+ indexkind = RELKIND_PARTITIONED_INDEX;
+
contype = *(PQgetvalue(res, j, i_contype));
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -7711,6 +7786,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
constrinfo->separate = true;
indxinfo[j].indexconstraint = constrinfo->dobj.dumpId;
+ if (relstats != NULL)
+ addObjectDependency(&relstats->dobj, constrinfo->dobj.dumpId);
}
else
{
@@ -10298,6 +10375,296 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
+/*
+ * Tabular description of the parameters to pg_restore_relation_stats()
+ * param_name, param_type
+ */
+static const char *rel_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"version", "integer"},
+ {"relpages", "integer"},
+ {"reltuples", "real"},
+ {"relallvisible", "integer"},
+};
+
+/*
+ * Tabular description of the parameters to pg_restore_attribute_stats()
+ * param_name, param_type
+ */
+static const char *att_stats_arginfo[][2] = {
+ {"relation", "regclass"},
+ {"attname", "name"},
+ {"inherited", "boolean"},
+ {"version", "integer"},
+ {"null_frac", "float4"},
+ {"avg_width", "integer"},
+ {"n_distinct", "float4"},
+ {"most_common_vals", "text"},
+ {"most_common_freqs", "float4[]"},
+ {"histogram_bounds", "text"},
+ {"correlation", "float4"},
+ {"most_common_elems", "text"},
+ {"most_common_elem_freqs", "float4[]"},
+ {"elem_count_histogram", "float4[]"},
+ {"range_length_histogram", "text"},
+ {"range_empty_frac", "float4"},
+ {"range_bounds_histogram", "text"},
+};
+
+/*
+ * getRelStatsExportQuery --
+ *
+ * Generate a query that will fetch all relation (e.g. pg_class)
+ * stats for a given relation.
+ */
+static void
+getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "current_setting('server_version_num') AS version, "
+ "c.relpages, c.reltuples, c.relallvisible "
+ "FROM pg_class c "
+ "JOIN pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE n.nspname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND c.relname = ");
+ appendStringLiteralAH(query, relname, fout);
+}
+
+/*
+ * getAttStatsExportQuery --
+ *
+ * Generate a query that will fetch all attribute (e.g. pg_statistic)
+ * stats for a given relation.
+ */
+static void
+getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
+ const char *schemaname, const char *relname)
+{
+ resetPQExpBuffer(query);
+ appendPQExpBufferStr(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "current_setting('server_version_num') AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,");
+
+ if (fout->remoteVersion >= 170000)
+ appendPQExpBufferStr(query,
+ "s.range_length_histogram,"
+ "s.range_empty_frac,"
+ "s.range_bounds_histogram ");
+ else
+ appendPQExpBufferStr(query,
+ "NULL AS range_length_histogram,"
+ "NULL AS range_empty_frac,"
+ "NULL AS range_bounds_histogram ");
+
+ appendPQExpBufferStr(query,
+ "FROM pg_stats s "
+ "JOIN pg_namespace n "
+ "ON n.nspname = s.schemaname "
+ "JOIN pg_class c "
+ "ON c.relname = s.tablename "
+ "AND c.relnamespace = n.oid "
+ "WHERE s.schemaname = ");
+ appendStringLiteralAH(query, schemaname, fout);
+ appendPQExpBufferStr(query, " AND s.tablename = ");
+ appendStringLiteralAH(query, relname, fout);
+ appendPQExpBufferStr(query, " ORDER BY s.attname, s.inherited");
+}
+
+
+/*
+ * appendNamedArgument --
+ *
+ * Convenience routine for constructing parameters of the form:
+ * 'paraname', 'value'::type
+ */
+static void
+appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
+ const char *argval, const char *argtype)
+{
+ appendPQExpBufferStr(out, "\t");
+
+ appendStringLiteralAH(out, argname, fout);
+ appendPQExpBufferStr(out, ", ");
+
+ appendStringLiteralAH(out, argval, fout);
+ appendPQExpBuffer(out, "::%s", argtype);
+}
+
+/*
+ * appendRelStatsImport --
+ *
+ * Append a formatted pg_restore_relation_stats statement.
+ */
+static void
+appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ const char *sep = "";
+
+ if (PQntuples(res) == 0)
+ return;
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
+
+ for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
+ {
+ const char *argname = rel_stats_arginfo[argno][0];
+ const char *argtype = rel_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("relation stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, 0, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
+
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+}
+
+/*
+ * appendAttStatsImport --
+ *
+ * Append a series of formatted pg_restore_attribute_stats statements.
+ */
+static void
+appendAttStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+{
+ for (int rownum = 0; rownum < PQntuples(res); rownum++)
+ {
+ const char *sep = "";
+
+ appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n");
+ for (int argno = 0; argno < lengthof(att_stats_arginfo); argno++)
+ {
+ const char *argname = att_stats_arginfo[argno][0];
+ const char *argtype = att_stats_arginfo[argno][1];
+ int fieldno = PQfnumber(res, argname);
+
+ if (fieldno < 0)
+ pg_fatal("attribute stats export query missing field '%s'",
+ argname);
+
+ if (PQgetisnull(res, rownum, fieldno))
+ continue;
+
+ appendPQExpBufferStr(out, sep);
+ appendNamedArgument(out, fout, argname, PQgetvalue(res, rownum, fieldno), argtype);
+ sep = ",\n";
+ }
+ appendPQExpBufferStr(out, "\n);\n");
+ }
+}
+
+/*
+ * Decide which section to use based on the relkind of the parent object.
+ *
+ * NB: materialized views may be postponed from SECTION_PRE_DATA to
+ * SECTION_POST_DATA to resolve some kinds of dependency problems. If so, the
+ * matview stats will also be postponed to SECTION_POST_DATA. See
+ * repairMatViewBoundaryMultiLoop().
+ */
+static teSection
+statisticsDumpSection(const RelStatsInfo *rsinfo)
+{
+ switch (rsinfo->relkind)
+ {
+ case RELKIND_RELATION:
+ case RELKIND_PARTITIONED_TABLE:
+ case RELKIND_MATVIEW:
+ return SECTION_DATA;
+ case RELKIND_INDEX:
+ case RELKIND_PARTITIONED_INDEX:
+ return SECTION_POST_DATA;
+ default:
+ pg_fatal("cannot dump statistics for relation kind '%c'",
+ rsinfo->relkind);
+ }
+
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * dumpRelationStats --
+ *
+ * Dump command to import stats into the relation on the new database.
+ */
+static void
+dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
+{
+ PGresult *res;
+ PQExpBuffer query;
+ PQExpBuffer out;
+ PQExpBuffer tag;
+ DumpableObject *dobj = (DumpableObject *) &rsinfo->dobj;
+ DumpId *deps = NULL;
+ int ndeps = 0;
+
+ /* nothing to do if we are not dumping statistics */
+ if (!fout->dopt->dumpStatistics)
+ return;
+
+ /* dependent on the relation definition, if doing schema */
+ if (fout->dopt->dumpSchema)
+ {
+ deps = dobj->dependencies;
+ ndeps = dobj->nDeps;
+ }
+
+ tag = createPQExpBuffer();
+ appendPQExpBufferStr(tag, fmtId(dobj->name));
+
+ query = createPQExpBuffer();
+ out = createPQExpBuffer();
+
+ getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendRelStatsImport(out, fout, res);
+ PQclear(res);
+
+ getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
+ dobj->name);
+ res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
+ appendAttStatsImport(out, fout, res);
+ PQclear(res);
+
+ ArchiveEntry(fout, nilCatalogId, createDumpId(),
+ ARCHIVE_OPTS(.tag = tag->data,
+ .namespace = dobj->namespace->dobj.name,
+ .description = "STATISTICS DATA",
+ .section = rsinfo->postponed_def ?
+ SECTION_POST_DATA : statisticsDumpSection(rsinfo),
+ .createStmt = out->data,
+ .deps = deps,
+ .nDeps = ndeps));
+
+ destroyPQExpBuffer(query);
+ destroyPQExpBuffer(out);
+ destroyPQExpBuffer(tag);
+}
+
/*
* dumpTableComment --
*
@@ -10746,6 +11113,9 @@ dumpDumpableObject(Archive *fout, DumpableObject *dobj)
case DO_SUBSCRIPTION_REL:
dumpSubscriptionTable(fout, (const SubRelInfo *) dobj);
break;
+ case DO_REL_STATS:
+ dumpRelationStats(fout, (const RelStatsInfo *) dobj);
+ break;
case DO_PRE_DATA_BOUNDARY:
case DO_POST_DATA_BOUNDARY:
/* never dumped, nothing to do */
@@ -18975,6 +19345,16 @@ addBoundaryDependencies(DumpableObject **dobjs, int numObjs,
/* must come after the pre-data boundary */
addObjectDependency(dobj, preDataBound->dumpId);
break;
+ case DO_REL_STATS:
+ /* stats section varies by parent object type, DATA or POST */
+ if (statisticsDumpSection((RelStatsInfo *) dobj) == SECTION_DATA)
+ {
+ addObjectDependency(dobj, preDataBound->dumpId);
+ addObjectDependency(postDataBound, dobj->dumpId);
+ }
+ else
+ addObjectDependency(dobj, postDataBound->dumpId);
+ break;
}
}
}
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 7139c88a69a..410c162a85d 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -83,10 +83,13 @@ typedef enum
DO_PUBLICATION,
DO_PUBLICATION_REL,
DO_PUBLICATION_TABLE_IN_SCHEMA,
+ DO_REL_STATS,
DO_SUBSCRIPTION,
DO_SUBSCRIPTION_REL, /* see note for SubRelInfo */
} DumpableObjectType;
+#define NUM_DUMPABLE_OBJECT_TYPES (DO_SUBSCRIPTION_REL + 1)
+
/*
* DumpComponents is a bitmask of the potentially dumpable components of
* a database object: its core definition, plus optional attributes such
@@ -110,6 +113,7 @@ typedef uint32 DumpComponents;
#define DUMP_COMPONENT_ACL (1 << 4)
#define DUMP_COMPONENT_POLICY (1 << 5)
#define DUMP_COMPONENT_USERMAP (1 << 6)
+#define DUMP_COMPONENT_STATISTICS (1 << 7)
#define DUMP_COMPONENT_ALL (0xFFFF)
/*
@@ -430,6 +434,13 @@ typedef struct _indexAttachInfo
IndxInfo *partitionIdx; /* link to index on partition */
} IndexAttachInfo;
+typedef struct _relStatsInfo
+{
+ DumpableObject dobj;
+ char relkind; /* 'r', 'm', 'i', etc */
+ bool postponed_def; /* stats must be postponed into post-data */
+} RelStatsInfo;
+
typedef struct _statsExtInfo
{
DumpableObject dobj;
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index dc9a28924bd..201eeedde73 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -81,6 +81,7 @@ enum dbObjectTypePriorities
PRIO_TABLE_DATA,
PRIO_SEQUENCE_SET,
PRIO_LARGE_OBJECT_DATA,
+ PRIO_STATISTICS_DATA_DATA,
PRIO_POST_DATA_BOUNDARY, /* boundary! */
PRIO_CONSTRAINT,
PRIO_INDEX,
@@ -148,11 +149,12 @@ static const int dbObjectTypePriority[] =
[DO_PUBLICATION] = PRIO_PUBLICATION,
[DO_PUBLICATION_REL] = PRIO_PUBLICATION_REL,
[DO_PUBLICATION_TABLE_IN_SCHEMA] = PRIO_PUBLICATION_TABLE_IN_SCHEMA,
+ [DO_REL_STATS] = PRIO_STATISTICS_DATA_DATA,
[DO_SUBSCRIPTION] = PRIO_SUBSCRIPTION,
[DO_SUBSCRIPTION_REL] = PRIO_SUBSCRIPTION_REL,
};
-StaticAssertDecl(lengthof(dbObjectTypePriority) == (DO_SUBSCRIPTION_REL + 1),
+StaticAssertDecl(lengthof(dbObjectTypePriority) == NUM_DUMPABLE_OBJECT_TYPES,
"array length mismatch");
static DumpId preDataBoundId;
@@ -801,11 +803,21 @@ repairMatViewBoundaryMultiLoop(DumpableObject *boundaryobj,
{
/* remove boundary's dependency on object after it in loop */
removeObjectDependency(boundaryobj, nextobj->dumpId);
- /* if that object is a matview, mark it as postponed into post-data */
+ /*
+ * If that object is a matview or matview status, mark it as postponed into
+ * post-data.
+ */
if (nextobj->objType == DO_TABLE)
{
TableInfo *nextinfo = (TableInfo *) nextobj;
+ if (nextinfo->relkind == RELKIND_MATVIEW)
+ nextinfo->postponed_def = true;
+ }
+ else if (nextobj->objType == DO_REL_STATS)
+ {
+ RelStatsInfo *nextinfo = (RelStatsInfo *) nextobj;
+
if (nextinfo->relkind == RELKIND_MATVIEW)
nextinfo->postponed_def = true;
}
@@ -1018,6 +1030,21 @@ repairDependencyLoop(DumpableObject **loop,
{
DumpableObject *nextobj;
+ nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
+ repairMatViewBoundaryMultiLoop(loop[j], nextobj);
+ return;
+ }
+ }
+ }
+ else if (loop[i]->objType == DO_REL_STATS &&
+ ((RelStatsInfo *) loop[i])->relkind == RELKIND_MATVIEW)
+ {
+ for (j = 0; j < nLoop; j++)
+ {
+ if (loop[j]->objType == DO_POST_DATA_BOUNDARY)
+ {
+ DumpableObject *nextobj;
+
nextobj = (j < nLoop - 1) ? loop[j + 1] : loop[0];
repairMatViewBoundaryMultiLoop(loop[j], nextobj);
return;
@@ -1500,6 +1527,11 @@ describeDumpableObject(DumpableObject *obj, char *buf, int bufsize)
"POST-DATA BOUNDARY (ID %d)",
obj->dumpId);
return;
+ case DO_REL_STATS:
+ snprintf(buf, bufsize,
+ "RELATION STATISTICS FOR %s (ID %d OID %u)",
+ obj->name, obj->dumpId, obj->catId.oid);
+ return;
}
/* shouldn't get here */
snprintf(buf, bufsize,
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index 64a60a26092..90847c7c8c7 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -103,6 +103,9 @@ static int use_setsessauth = 0;
static int no_comments = 0;
static int no_publications = 0;
static int no_security_labels = 0;
+static int no_data = 0;
+static int no_schema = 0;
+static int no_statistics = 0;
static int no_subscriptions = 0;
static int no_toast_compression = 0;
static int no_unlogged_table_data = 0;
@@ -139,6 +142,7 @@ main(int argc, char *argv[])
{"port", required_argument, NULL, 'p'},
{"roles-only", no_argument, NULL, 'r'},
{"schema-only", no_argument, NULL, 's'},
+ {"statistics-only", no_argument, NULL, 'X'},
{"superuser", required_argument, NULL, 'S'},
{"tablespaces-only", no_argument, NULL, 't'},
{"username", required_argument, NULL, 'U'},
@@ -168,10 +172,13 @@ main(int argc, char *argv[])
{"role", required_argument, NULL, 3},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
{"no-role-passwords", no_argument, &no_role_passwords, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"no-sync", no_argument, NULL, 4},
{"no-toast-compression", no_argument, &no_toast_compression, 1},
{"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1},
@@ -328,6 +335,10 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " -x");
break;
+ case 'X':
+ appendPQExpBufferStr(pgdumpopts, " -X");
+ break;
+
case 0:
break;
@@ -447,10 +458,16 @@ main(int argc, char *argv[])
appendPQExpBufferStr(pgdumpopts, " --use-set-session-authorization");
if (no_comments)
appendPQExpBufferStr(pgdumpopts, " --no-comments");
+ if (no_data)
+ appendPQExpBufferStr(pgdumpopts, " --no-data");
if (no_publications)
appendPQExpBufferStr(pgdumpopts, " --no-publications");
if (no_security_labels)
appendPQExpBufferStr(pgdumpopts, " --no-security-labels");
+ if (no_schema)
+ appendPQExpBufferStr(pgdumpopts, " --no-schema");
+ if (no_statistics)
+ appendPQExpBufferStr(pgdumpopts, " --no-statistics");
if (no_subscriptions)
appendPQExpBufferStr(pgdumpopts, " --no-subscriptions");
if (no_toast_compression)
@@ -667,6 +684,7 @@ help(void)
printf(_(" --no-publications do not dump publications\n"));
printf(_(" --no-role-passwords do not dump passwords for roles\n"));
printf(_(" --no-security-labels do not dump security label assignments\n"));
+ printf(_(" --no-statistics do not dump statistics\n"));
printf(_(" --no-subscriptions do not dump subscriptions\n"));
printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" --no-table-access-method do not dump table access methods\n"));
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index c602272d7db..02af89bae15 100644
--- a/src/bin/pg_dump/pg_restore.c
+++ b/src/bin/pg_dump/pg_restore.c
@@ -63,6 +63,9 @@ main(int argc, char **argv)
int numWorkers = 1;
Archive *AH;
char *inputFileSpec;
+ bool data_only = false;
+ bool schema_only = false;
+ bool statistics_only = false;
static int disable_triggers = 0;
static int enable_row_security = 0;
static int if_exists = 0;
@@ -71,12 +74,13 @@ main(int argc, char **argv)
static int outputNoTablespaces = 0;
static int use_setsessauth = 0;
static int no_comments = 0;
+ static int no_data = 0;
static int no_publications = 0;
+ static int no_schema = 0;
static int no_security_labels = 0;
+ static int no_statistics = 0;
static int no_subscriptions = 0;
static int strict_names = 0;
- bool data_only = false;
- bool schema_only = false;
struct option cmdopts[] = {
{"clean", 0, NULL, 'c'},
@@ -108,6 +112,7 @@ main(int argc, char **argv)
{"username", 1, NULL, 'U'},
{"verbose", 0, NULL, 'v'},
{"single-transaction", 0, NULL, '1'},
+ {"statistics-only", no_argument, NULL, 'X'},
/*
* the following options don't have an equivalent short option letter
@@ -124,9 +129,12 @@ main(int argc, char **argv)
{"transaction-size", required_argument, NULL, 5},
{"use-set-session-authorization", no_argument, &use_setsessauth, 1},
{"no-comments", no_argument, &no_comments, 1},
+ {"no-data", no_argument, &no_data, 1},
{"no-publications", no_argument, &no_publications, 1},
+ {"no-schema", no_argument, &no_schema, 1},
{"no-security-labels", no_argument, &no_security_labels, 1},
{"no-subscriptions", no_argument, &no_subscriptions, 1},
+ {"no-statistics", no_argument, &no_statistics, 1},
{"filter", required_argument, NULL, 4},
{NULL, 0, NULL, 0}
@@ -271,6 +279,10 @@ main(int argc, char **argv)
opts->aclsSkip = 1;
break;
+ case 'X': /* Restore statistics only */
+ statistics_only = true;
+ break;
+
case '1': /* Restore data in a single transaction */
opts->single_txn = true;
opts->exit_on_error = true;
@@ -343,6 +355,10 @@ main(int argc, char **argv)
if (data_only && schema_only)
pg_fatal("options -s/--schema-only and -a/--data-only cannot be used together");
+ if (data_only && statistics_only)
+ pg_fatal("options -a/--data-only and -X/--statistics-only cannot be used together");
+ if (schema_only && statistics_only)
+ pg_fatal("options -s/--schema-only and -X/--statistics-only cannot be used together");
if (data_only && opts->dropSchema)
pg_fatal("options -c/--clean and -a/--data-only cannot be used together");
@@ -362,8 +378,9 @@ main(int argc, char **argv)
pg_fatal("cannot specify both --single-transaction and multiple jobs");
/* set derivative flags */
- opts->dumpSchema = (!data_only);
- opts->dumpData = (!schema_only);
+ opts->dumpData = data_only || (!no_data && !schema_only && !statistics_only);
+ opts->dumpSchema = schema_only || (!no_schema && !data_only && !statistics_only);
+ opts->dumpStatistics = statistics_only || (!no_statistics && !data_only && !schema_only);
opts->disable_triggers = disable_triggers;
opts->enable_row_security = enable_row_security;
@@ -375,6 +392,8 @@ main(int argc, char **argv)
opts->no_publications = no_publications;
opts->no_security_labels = no_security_labels;
opts->no_subscriptions = no_subscriptions;
+ opts->no_statistics = no_statistics;
+ opts->no_data = no_data;
if (if_exists && !opts->dropSchema)
pg_fatal("option --if-exists requires option -c/--clean");
@@ -482,6 +501,7 @@ usage(const char *progname)
printf(_(" -t, --table=NAME restore named relation (table, view, etc.)\n"));
printf(_(" -T, --trigger=NAME restore named trigger\n"));
printf(_(" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n"));
+ printf(_(" -X, --statistics-only restore only the statistics, not schema or data\n"));
printf(_(" -1, --single-transaction restore as a single transaction\n"));
printf(_(" --disable-triggers disable triggers during data-only restore\n"));
printf(_(" --enable-row-security enable row security\n"));
@@ -489,10 +509,13 @@ usage(const char *progname)
" in FILENAME\n"));
printf(_(" --if-exists use IF EXISTS when dropping objects\n"));
printf(_(" --no-comments do not restore comment commands\n"));
+ printf(_(" --no-data do not restore data\n"));
printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n"
" created\n"));
printf(_(" --no-publications do not restore publications\n"));
+ printf(_(" --no-schema do not restore schema\n"));
printf(_(" --no-security-labels do not restore security labels\n"));
+ printf(_(" --no-statistics do not restore statistics\n"));
printf(_(" --no-subscriptions do not restore subscriptions\n"));
printf(_(" --no-table-access-method do not restore table access methods\n"));
printf(_(" --no-tablespaces do not restore tablespace assignments\n"));
diff --git a/src/bin/pg_dump/t/001_basic.pl b/src/bin/pg_dump/t/001_basic.pl
index 214240f1ae5..f29da06ed28 100644
--- a/src/bin/pg_dump/t/001_basic.pl
+++ b/src/bin/pg_dump/t/001_basic.pl
@@ -50,12 +50,30 @@ command_fails_like(
'pg_dump: options -s/--schema-only and -a/--data-only cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '-s', '-X' ],
+ qr/\Qpg_dump: error: options -s\/--schema-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -s/--schema-only and -X/--statistics-only cannot be used together'
+);
+
+command_fails_like(
+ [ 'pg_dump', '-a', '-X' ],
+ qr/\Qpg_dump: error: options -a\/--data-only and -X\/--statistics-only cannot be used together\E/,
+ 'pg_dump: error: options -a/--data-only and -X/--statistics-only cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-s', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: options -s\/--schema-only and --include-foreign-data cannot be used together\E/,
'pg_dump: options -s/--schema-only and --include-foreign-data cannot be used together'
);
+command_fails_like(
+ [ 'pg_dump', '--statistics-only', '--no-statistics' ],
+ qr/\Qpg_dump: error: options -X\/--statistics-only and --no-statistics cannot be used together\E/,
+ 'pg_dump: options -X\/--statistics-only and --no-statistics cannot be used together'
+);
+
command_fails_like(
[ 'pg_dump', '-j2', '--include-foreign-data=xxx' ],
qr/\Qpg_dump: error: option --include-foreign-data is not supported with parallel backup\E/,
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index bc5d9222a20..e1545e797a7 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -65,7 +65,7 @@ my %pgdump_runs = (
'--format' => 'custom',
'--file' => "$tempdir/binary_upgrade.dump",
'--no-password',
- '--schema-only',
+ '--no-data',
'--binary-upgrade',
'--dbname' => 'postgres', # alternative way to specify database
],
@@ -710,6 +710,39 @@ my %pgdump_runs = (
'--no-large-objects',
'postgres',
],
+ },
+ no_statistics => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_statistics.sql",
+ '--no-statistics',
+ 'postgres',
+ ],
+ },
+ no_data_no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_data_no_schema.sql",
+ '--no-data',
+ '--no-schema',
+ 'postgres',
+ ],
+ },
+ statistics_only => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/statistics_only.sql",
+ '--statistics-only',
+ 'postgres',
+ ],
+ },
+ no_schema => {
+ dump_cmd => [
+ 'pg_dump', '--no-sync',
+ "--file=$tempdir/no_schema.sql",
+ '--no-schema',
+ 'postgres',
+ ],
},);
###############################################################
@@ -776,6 +809,7 @@ my %full_runs = (
no_large_objects => 1,
no_owner => 1,
no_privs => 1,
+ no_statistics => 1,
no_table_access_method => 1,
pg_dumpall_dbprivs => 1,
pg_dumpall_exclude => 1,
@@ -977,6 +1011,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1390,6 +1425,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1411,6 +1447,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1432,6 +1469,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1598,6 +1636,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
},
@@ -1751,6 +1790,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_test_table => 1,
section_data => 1,
},
@@ -1778,6 +1818,7 @@ my %tests = (
data_only => 1,
exclude_test_table => 1,
exclude_test_table_data => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1798,7 +1839,10 @@ my %tests = (
\QCOPY dump_test.fk_reference_test_table (col1) FROM stdin;\E
\n(?:\d\n){5}\\\.\n
/xms,
- like => { data_only => 1, },
+ like => {
+ data_only => 1,
+ no_schema => 1,
+ },
},
'COPY test_second_table' => {
@@ -1814,6 +1858,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1836,6 +1881,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1859,6 +1905,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1881,6 +1928,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -1903,6 +1951,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
},
unlike => {
@@ -3299,6 +3348,7 @@ my %tests = (
like => {
%full_runs,
data_only => 1,
+ no_schema => 1,
section_data => 1,
only_dump_test_schema => 1,
test_schema_plus_large_objects => 1,
@@ -3469,6 +3519,7 @@ my %tests = (
%full_runs,
%dump_test_schema_runs,
data_only => 1,
+ no_schema => 1,
only_dump_measurement => 1,
section_data => 1,
only_dump_test_schema => 1,
@@ -4353,6 +4404,7 @@ my %tests = (
column_inserts => 1,
data_only => 1,
inserts => 1,
+ no_schema => 1,
section_data => 1,
test_schema_plus_large_objects => 1,
binary_upgrade => 1,
@@ -4653,6 +4705,61 @@ my %tests = (
},
},
+ #
+ # TABLE and MATVIEW stats will end up in SECTION_DATA.
+ # INDEX stats (expression columns only) will end up in SECTION_POST_DATA.
+ #
+ 'statistics_import' => {
+ create_sql => '
+ CREATE TABLE dump_test.has_stats
+ AS SELECT g.g AS x, g.g / 2 AS y FROM generate_series(1,100) AS g(g);
+ CREATE MATERIALIZED VIEW dump_test.has_stats_mv AS SELECT * FROM dump_test.has_stats;
+ CREATE INDEX dup_test_post_data_ix ON dump_test.has_stats((x - 1));
+ ANALYZE dump_test.has_stats, dump_test.has_stats_mv;',
+ regexp => qr/pg_catalog.pg_restore_attribute_stats/,
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ exclude_dump_test_schema => 1,
+ no_statistics => 1,
+ only_dump_measurement => 1,
+ schema_only => 1,
+ },
+ },
+
+ #
+ # While attribute stats (aka pg_statistic stats) only appear for tables
+ # that have been analyzed, all tables will have relation stats because
+ # those come from pg_class.
+ #
+ 'relstats_on_unanalyzed_tables' => {
+ regexp => qr/pg_catalog.pg_restore_relation_stats/,
+
+ like => {
+ %full_runs,
+ %dump_test_schema_runs,
+ no_data_no_schema => 1,
+ no_schema => 1,
+ only_dump_test_table => 1,
+ role => 1,
+ role_parallel => 1,
+ section_data => 1,
+ section_post_data => 1,
+ statistics_only => 1,
+ },
+ unlike => {
+ no_statistics => 1,
+ schema_only => 1,
+ },
+ },
+
# CREATE TABLE with partitioned table and various AMs. One
# partition uses the same default as the parent, and a second
# uses its own AM.
diff --git a/src/bin/pg_upgrade/dump.c b/src/bin/pg_upgrade/dump.c
index 8ce0fa3020e..7d0871e3c5e 100644
--- a/src/bin/pg_upgrade/dump.c
+++ b/src/bin/pg_upgrade/dump.c
@@ -21,8 +21,8 @@ generate_old_dump(void)
/* run new pg_dumpall binary for globals */
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
- "\"%s/pg_dumpall\" %s --globals-only --quote-all-identifiers "
- "--binary-upgrade %s --no-sync -f \"%s/%s\"",
+ "\"%s/pg_dumpall\" %s %s --globals-only --quote-all-identifiers "
+ "--binary-upgrade --no-sync -f \"%s/%s\"",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
log_opts.dumpdir,
@@ -52,10 +52,11 @@ generate_old_dump(void)
snprintf(log_file_name, sizeof(log_file_name), DB_DUMP_LOG_FILE_MASK, old_db->db_oid);
parallel_exec_prog(log_file_name, NULL,
- "\"%s/pg_dump\" %s --schema-only --quote-all-identifiers "
+ "\"%s/pg_dump\" %s --no-data %s --quote-all-identifiers "
"--binary-upgrade --format=custom %s --no-sync --file=\"%s/%s\" %s",
new_cluster.bindir, cluster_conn_opts(&old_cluster),
log_opts.verbose ? "--verbose" : "",
+ user_opts.do_statistics ? "" : "--no-statistics",
log_opts.dumpdir,
sql_file_name, escaped_connstr.data);
diff --git a/src/bin/pg_upgrade/option.c b/src/bin/pg_upgrade/option.c
index 108eb7a1ba4..3b6c7ec994e 100644
--- a/src/bin/pg_upgrade/option.c
+++ b/src/bin/pg_upgrade/option.c
@@ -60,6 +60,8 @@ parseCommandLine(int argc, char *argv[])
{"copy", no_argument, NULL, 2},
{"copy-file-range", no_argument, NULL, 3},
{"sync-method", required_argument, NULL, 4},
+ {"with-statistics", no_argument, NULL, 5},
+ {"no-statistics", no_argument, NULL, 6},
{NULL, 0, NULL, 0}
};
@@ -70,6 +72,7 @@ parseCommandLine(int argc, char *argv[])
user_opts.do_sync = true;
user_opts.transfer_mode = TRANSFER_MODE_COPY;
+ user_opts.do_statistics = true;
os_info.progname = get_progname(argv[0]);
@@ -212,6 +215,13 @@ parseCommandLine(int argc, char *argv[])
user_opts.sync_method = pg_strdup(optarg);
break;
+ case 5:
+ user_opts.do_statistics = true;
+ break;
+ case 6:
+ user_opts.do_statistics = false;
+ break;
+
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
os_info.progname);
@@ -306,7 +316,9 @@ usage(void)
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" --copy copy files to new cluster (default)\n"));
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
+ printf(_(" --no-statistics do not import statistics from old cluster\n"));
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --with-statistics import statistics from old cluster (default)\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"
diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h
index 0cdd675e4f1..3fe111fbde5 100644
--- a/src/bin/pg_upgrade/pg_upgrade.h
+++ b/src/bin/pg_upgrade/pg_upgrade.h
@@ -327,6 +327,7 @@ typedef struct
int jobs; /* number of processes/threads to use */
char *socketdir; /* directory to use for Unix sockets */
char *sync_method;
+ bool do_statistics; /* carry over statistics from old cluster */
} UserOpts;
typedef struct
diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 24fcc76d72c..0d5e7d5a2d7 100644
--- a/doc/src/sgml/ref/pg_dump.sgml
+++ b/doc/src/sgml/ref/pg_dump.sgml
@@ -123,7 +123,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are dumped.
</para>
@@ -141,13 +141,11 @@ PostgreSQL documentation
<listitem>
<para>
Include large objects in the dump. This is the default behavior
- except when <option>--schema</option>, <option>--table</option>, or
- <option>--schema-only</option> is specified. The <option>-b</option>
- switch is therefore only useful to add large objects to dumps
- where a specific schema or table has been requested. Note that
- large objects are considered data and therefore will be included when
- <option>--data-only</option> is used, but not
- when <option>--schema-only</option> is.
+ except when <option>--schema</option>, <option>--table</option>,
+ <option>--schema-only</option>, or <option>--statistics-only</option>, or
+ <option>--no-data</option> is specified. The <option>-b</option>
+ switch is therefore only useful to add large objects to dumps where a
+ specific schema or table has been requested.
</para>
</listitem>
</varlistentry>
@@ -516,10 +514,11 @@ PostgreSQL documentation
<term><option>--schema-only</option></term>
<listitem>
<para>
- Dump only the object definitions (schema), not data.
+ Dump only the object definitions (schema), not data or statistics.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive to <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
<option>--section=pre-data --section=post-data</option>.
@@ -652,6 +651,18 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
<term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
@@ -741,7 +752,8 @@ PostgreSQL documentation
<term><option>--disable-triggers</option></term>
<listitem>
<para>
- This option is relevant only when creating a data-only dump.
+ This option is relevant only when creating a dump that includes data
+ but does not include schema.
It instructs <application>pg_dump</application> to include commands
to temporarily disable triggers on the target tables while
the data is restored. Use this if you have referential
@@ -824,16 +836,17 @@ PostgreSQL documentation
<term><option>--exclude-table-data=<replaceable class="parameter">pattern</replaceable></option></term>
<listitem>
<para>
- Do not dump data for any tables matching <replaceable
+ Do not dump data or statistics for any tables matching <replaceable
class="parameter">pattern</replaceable>. The pattern is
interpreted according to the same rules as for <option>-t</option>.
<option>--exclude-table-data</option> can be given more than once to
- exclude tables matching any of several patterns. This option is
- useful when you need the definition of a particular table even
- though you do not need the data in it.
+ exclude tables matching any of several patterns. This option is useful
+ when you need the definition of a particular table even though you do
+ not need the data in it.
</para>
<para>
- To exclude data for all tables in the database, see <option>--schema-only</option>.
+ To exclude data for all tables in the database, see <option>--schema-only</option>
+ or <option>--statistics-only</option>.
</para>
</listitem>
</varlistentry>
@@ -1080,6 +1093,15 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -1098,6 +1120,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
@@ -1236,9 +1276,11 @@ PostgreSQL documentation
</para>
<para>
The data section contains actual table data, large-object
- contents, and sequence values.
+ contents, statitistics for tables and materialized views and
+ sequence values.
Post-data items include definitions of indexes, triggers, rules,
- and constraints other than validated check constraints.
+ statistics for indexes, and constraints other than validated check
+ constraints.
Pre-data items include all other data definition items.
</para>
</listitem>
@@ -1581,7 +1623,7 @@ CREATE DATABASE foo WITH TEMPLATE template0;
</para>
<para>
- When a data-only dump is chosen and the option <option>--disable-triggers</option>
+ When a dump without schema is chosen and the option <option>--disable-triggers</option>
is used, <application>pg_dump</application> emits commands
to disable triggers on user tables before inserting the data,
and then commands to re-enable them after the data has been
diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml
index 39d93c2c0e3..15fb40e7be9 100644
--- a/doc/src/sgml/ref/pg_dumpall.sgml
+++ b/doc/src/sgml/ref/pg_dumpall.sgml
@@ -81,7 +81,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Dump only the data, not the schema (data definitions).
+ Dump only the data, not the schema (data definitions) or statistics.
</para>
</listitem>
</varlistentry>
@@ -265,6 +265,17 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Dump only the statistics, not the schema (data definitions) or data.
+ Statistics for tables, materialized views, and indexes are dumped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--binary-upgrade</option></term>
<listitem>
@@ -307,7 +318,7 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
<term><option>--disable-triggers</option></term>
<listitem>
<para>
- This option is relevant only when creating a data-only dump.
+ This option is relevant only when creating a dump with data and without schema.
It instructs <application>pg_dumpall</application> to include commands
to temporarily disable triggers on the target tables while
the data is restored. Use this if you have referential
@@ -422,6 +433,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not dump data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-publications</option></term>
<listitem>
@@ -447,6 +467,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not dump schema (data definitions).
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -456,6 +485,15 @@ exclude database <replaceable class="parameter">PATTERN</replaceable>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not dump statistics.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml
index b8b27e1719e..86e4264714d 100644
--- a/doc/src/sgml/ref/pg_restore.sgml
+++ b/doc/src/sgml/ref/pg_restore.sgml
@@ -94,7 +94,7 @@ PostgreSQL documentation
<term><option>--data-only</option></term>
<listitem>
<para>
- Restore only the data, not the schema (data definitions).
+ Restore only the data, not the schema (data definitions) or statistics.
Table data, large objects, and sequence values are restored,
if present in the archive.
</para>
@@ -483,10 +483,11 @@ PostgreSQL documentation
to the extent that schema entries are present in the archive.
</para>
<para>
- This option is the inverse of <option>--data-only</option>.
+ This option is mutually exclusive of <option>--data-only</option>
+ and <option>--statistics-only</option>.
It is similar to, but for historical reasons not identical to,
specifying
- <option>--section=pre-data --section=post-data</option>.
+ <option>--section=pre-data --section=post-data --no-statistics</option>.
</para>
<para>
(Do not confuse this with the <option>--schema</option> option, which
@@ -599,6 +600,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-X</option></term>
+ <term><option>--statistics-only</option></term>
+ <listitem>
+ <para>
+ Restore only the statistics, not schema (data definitions) or data.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-1</option></term>
<term><option>--single-transaction</option></term>
@@ -617,7 +628,7 @@ PostgreSQL documentation
<term><option>--disable-triggers</option></term>
<listitem>
<para>
- This option is relevant only when performing a data-only restore.
+ This option is relevant only when performing a restore without schema.
It instructs <application>pg_restore</application> to execute commands
to temporarily disable triggers on the target tables while
the data is restored. Use this if you have referential
@@ -681,6 +692,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-data</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore data, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-data-for-failed-tables</option></term>
<listitem>
@@ -713,6 +734,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-schema</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore schema (data definitions), even if
+ the archive contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-security-labels</option></term>
<listitem>
@@ -723,6 +754,16 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not output commands to restore statistics, even if the archive
+ contains them.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--no-subscriptions</option></term>
<listitem>
diff --git a/doc/src/sgml/ref/pgupgrade.sgml b/doc/src/sgml/ref/pgupgrade.sgml
index 4777381dac2..64a1ebd613b 100644
--- a/doc/src/sgml/ref/pgupgrade.sgml
+++ b/doc/src/sgml/ref/pgupgrade.sgml
@@ -145,6 +145,24 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>--with-statistics</option></term>
+ <listitem>
+ <para>
+ Restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term><option>--no-statistics</option></term>
+ <listitem>
+ <para>
+ Do not restore statistics from the old cluster into the new cluster.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-o</option> <replaceable class="parameter">options</replaceable></term>
<term><option>--old-options</option> <replaceable class="parameter">options</replaceable></term>
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index b6c170ac249..534481ed8a3 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2401,6 +2401,7 @@ RelMapFile
RelMapping
RelOptInfo
RelOptKind
+RelStatsInfo
RelToCheck
RelToCluster
RelabelType
base-commit: 4dd09a1d415d143386d5e6dc9519615a8b18f850
--
2.48.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-13 03:00 ` Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2025-02-13 03:00 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Tue, 2025-02-11 at 14:02 -0500, Corey Huinker wrote:
>
>
> The previous 0001 is now committed (thanks!) so only one remains.
>
Summary of the decisions made in this thread:
* pg_dump --data-only does not include stats[1]. This behavior was
not fully resolved, but I didn't see a reasonable set of options
where (a) --data-only would include stats; and (b) we could
specify what pg_upgrade needs, which is schema+stats. Jian seemed
to agree. However, this leaves us with the behavior where
--data-only doesn't get everything in SECTION_DATA, which might be
undesirable.
* stats are in SECTION_DATA[2], except for stats on objects that
are created in SECTION_POST_DATA, in which case the stats are
also in SECTION_POST_DATA
- indexes are created in SECTION_POST_DATA, and though plain
indexes don't have stats, expression indexes do
- MVs are normally created in SECTION_PRE_DATA, in which case
the stats are in SECTION_DATA; but MVs can be deferred to
SECTION_POST_DATA due to dependency on a primary key
* SECTION_NONE was proposed, but rejected[2]
* The default is to include the stats.[3]
* pg_dump[all] options are designed to allow specifying any
combination of schema[4], data, and stats:
--schema-only (schema), --no-schema (data+stats)
--data-only (data), --no-data (schema+stats)
--stats-only (stats), --no-stats (schema+data)
* A SECTION_STATS was proposed and rejected due to complexity[5]
* The prefix in the dump output will be "Statistics for " (instead
of "Data for ")[6]
* The TOC description will be "STATISTICS DATA", differentiating
it from an extended statistics object[6]
* pg_upgrade will now pass --no-data (schema+stats) to pg_dump
instead of --schema-only, thereby transferring the stats to the
new cluster[7]
It's been a long thread, so please tell me if I missed something or if
something needs more discussion.
I'm still reviewing v48, but I intend to commit something soon.
Regards,
Jeff Davis
[1]
https://www.postgresql.org/message-id/[email protected]
[2]
https://www.postgresql.org/message-id/1798867.1712376328%40sss.pgh.pa.us
[3]
https://www.postgresql.org/message-id/3228677.1713844341%40sss.pgh.pa.us
[4]
https://www.postgresql.org/message-id/CACJufxG6K4EAUROhdr0wkzMh5QyFLmdLZeAoh7Vh0-VbuAtHcw%40mail.gma...
[5]
https://www.postgresql.org/message-id/3156140.1713817153%40sss.pgh.pa.us
[6]
https://www.postgresql.org/message-id/[email protected]
[7]
https://www.postgresql.org/message-id/[email protected]
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-20 09:39 ` Jeff Davis <[email protected]>
2025-02-20 10:24 ` Re: Statistics Import and Export vignesh C <[email protected]>
2025-02-20 15:43 ` Re: Statistics Import and Export Andrew Dunstan <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
0 siblings, 3 replies; 81+ messages in thread
From: Jeff Davis @ 2025-02-20 09:39 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Wed, 2025-02-12 at 19:00 -0800, Jeff Davis wrote:
> I'm still reviewing v48, but I intend to commit something soon.
Committed with some revisions on top of v48:
* removed the short option -X, leaving the long option "--statistics-
only" with the same meaning.
* removed the redundant --with-statistics option for pg_upgrade,
because that's the default anyway.
* removed an unnecessary enum TocEntryType and cleaned up the API to
just pass the desired prefix directly to _printTocEntry().
* stabilized the 002_pg_upgrade test by turning off autovacuum before
the first pg_dumpall (we still want it to run before that to collect
stats).
* stabilized the 027_stream_regress recovery test by specifying --no-
statistics when comparing the data on primary and standby
* fixed the cross-version upgrade tests by using the
adjust_old_dumpfile to replace the version specifier with 000000 in the
argument list to pg_restore_* functions.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-20 10:24 ` vignesh C <[email protected]>
2 siblings, 0 replies; 81+ messages in thread
From: vignesh C @ 2025-02-20 10:24 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Thu, 20 Feb 2025 at 15:09, Jeff Davis <[email protected]> wrote:
>
> On Wed, 2025-02-12 at 19:00 -0800, Jeff Davis wrote:
> > I'm still reviewing v48, but I intend to commit something soon.
>
> Committed with some revisions on top of v48:
I was checking buildfarm for another commit of mine, while checking I
noticed there is a failure in crake at [1]. I felt it might be
related to this commit as it had passed with earlier runs:
--- /home/andrew/bf/root/upgrade.crake/HEAD/origin-REL9_2_STABLE.sql.fixed
2025-02-20 04:43:40.461092087 -0500
+++ /home/andrew/bf/root/upgrade.crake/HEAD/converted-REL9_2_STABLE-to-HEAD.sql.fixed
2025-02-20 04:43:40.463092092 -0500
@@ -184,21 +184,87 @@
--
SELECT * FROM pg_catalog.pg_restore_relation_stats(
'relation', '"MySchema"."Foo"'::regclass,
- 'version', '90224'::integer,
- 'relpages', '0'::integer,
- 'reltuples', '0'::real,
+ 'version', '000000'::integer,
+ 'relpages', '1'::integer,
+ 'reltuples', '1'::real,
'relallvisible', '0'::integer
[1] - https://buildfarm.postgresql.org/cgi-bin/show_stage_log.pl?nm=crake&dt=2025-02-20%2009%3A32%3A03...
Regards,
Vignesh
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-20 15:43 ` Andrew Dunstan <[email protected]>
2025-02-21 07:24 ` RE: Statistics Import and Export Hayato Kuroda (Fujitsu) <[email protected]>
2 siblings, 1 reply; 81+ messages in thread
From: Andrew Dunstan @ 2025-02-20 15:43 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; +Cc: Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On 2025-02-20 Th 4:39 AM, Jeff Davis wrote:
> On Wed, 2025-02-12 at 19:00 -0800, Jeff Davis wrote:
>> I'm still reviewing v48, but I intend to commit something soon.
> Committed with some revisions on top of v48:
>
> * removed the short option -X, leaving the long option "--statistics-
> only" with the same meaning.
>
> * removed the redundant --with-statistics option for pg_upgrade,
> because that's the default anyway.
>
> * removed an unnecessary enum TocEntryType and cleaned up the API to
> just pass the desired prefix directly to _printTocEntry().
>
> * stabilized the 002_pg_upgrade test by turning off autovacuum before
> the first pg_dumpall (we still want it to run before that to collect
> stats).
>
> * stabilized the 027_stream_regress recovery test by specifying --no-
> statistics when comparing the data on primary and standby
>
> * fixed the cross-version upgrade tests by using the
> adjust_old_dumpfile to replace the version specifier with 000000 in the
> argument list to pg_restore_* functions.
>
The buildfarm doesn't like with this.
<https://buildfarm.postgresql.org/cgi-bin/show_stage_log.pl?nm=crake&dt=2025-02-20%2009%3A32%3A03...;
The conversion regexes are wrong for versions < 10, where the major
version is '9.x', but that just seems to be the tip of the iceberg.
cheers
andrew
--
Andrew Dunstan
EDB: https://www.enterprisedb.com
^ permalink raw reply [nested|flat] 81+ messages in thread
* RE: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 15:43 ` Re: Statistics Import and Export Andrew Dunstan <[email protected]>
@ 2025-02-21 07:24 ` Hayato Kuroda (Fujitsu) <[email protected]>
2025-02-22 18:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Hayato Kuroda (Fujitsu) @ 2025-02-21 07:24 UTC (permalink / raw)
To: 'Andrew Dunstan' <[email protected]>; Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; +Cc: Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected] <[email protected]>
Dear members,
I hope I'm in the correct thread. I found the commit 1fd1bd8 - it is so cool.
I have a question for it.
ISTM commit message said that no need to do ANALYZE again.
```
Add support to pg_dump for dumping stats, and use that during
pg_upgrade so that statistics are transferred during upgrade. In most
cases this removes the need for a costly re-analyze after upgrade.
```
But pgupgrade.sgml [2] and source code [3] said that statistics must be updated.
Did I miss something, or you have been updating this?
[2]:
```
Because optimizer statistics are not transferred by <command>pg_upgrade</command>, you will
be instructed to run a command to regenerate that information at the end
of the upgrade. You might need to set connection parameters to
match your new cluster.
```
[3]:
```
pg_log(PG_REPORT,
"Optimizer statistics are not transferred by pg_upgrade.\n"
"Once you start the new server, consider running:\n"
" %s/vacuumdb %s--all --analyze-in-stages", new_cluster.bindir, user_specification.data);
```
Best regards,
Hayato Kuroda
FUJITSU LIMITED
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 15:43 ` Re: Statistics Import and Export Andrew Dunstan <[email protected]>
2025-02-21 07:24 ` RE: Statistics Import and Export Hayato Kuroda (Fujitsu) <[email protected]>
@ 2025-02-22 18:39 ` Jeff Davis <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Jeff Davis @ 2025-02-22 18:39 UTC (permalink / raw)
To: Hayato Kuroda (Fujitsu) <[email protected]>; 'Andrew Dunstan' <[email protected]>; Corey Huinker <[email protected]>; +Cc: Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected] <[email protected]>
On Fri, 2025-02-21 at 07:24 +0000, Hayato Kuroda (Fujitsu) wrote:
> I hope I'm in the correct thread. I found the commit 1fd1bd8 - it is
> so cool.
Yes, documentation corrections are appreciated, thank you.
> But pgupgrade.sgml [2] and source code [3] said that statistics must
> be updated.
Changed.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-21 20:23 ` Andres Freund <[email protected]>
2025-02-21 20:49 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2 siblings, 2 replies; 81+ messages in thread
From: Andres Freund @ 2025-02-21 20:23 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Hi,
On 2025-02-20 01:39:34 -0800, Jeff Davis wrote:
> Committed with some revisions on top of v48:
This made the pg_upgrade tests considerably slower.
In an assert build without optimization (since that's what I use for normal
dev work):
1fd1bd87101^ 65.03s
1fd1bd87101 86.84s
Looking at the times in the in the regress_log, I see:
good: [15:17:31.278](36.851s) ok 5 - regression tests pass
bad: [15:15:37.857](37.437s) ok 5 - regression tests pass
good: [15:17:36.721](5.436s) ok 6 - dump before running pg_upgrade
bad: [15:15:50.845](12.980s) ok 6 - dump before running pg_upgrade
good: [15:17:39.759](2.441s) ok 12 - run of pg_upgrade --check for new instance
bad: [15:15:53.861](2.415s) ok 12 - run of pg_upgrade --check for new instance
good: [15:17:51.249](11.489s) ok 14 - run of pg_upgrade for new instance
bad: [15:16:13.304](19.443s) ok 14 - run of pg_upgrade for new instance
good: [15:17:55.382](3.958s) ok 17 - dump after running pg_upgrade
bad: [15:16:23.766](10.290s) ok 17 - dump after running pg_upgrade
Which to me rather strongly suggests pg_dump has gotten a *lot* slower with
this change.
Greetings,
Andres
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
@ 2025-02-21 20:49 ` Tom Lane <[email protected]>
2025-02-21 21:11 ` Re: Statistics Import and Export Andres Freund <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Tom Lane @ 2025-02-21 20:49 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Andres Freund <[email protected]> writes:
> Which to me rather strongly suggests pg_dump has gotten a *lot* slower with
> this change.
Well, it's doing strictly more work, so somewhat slower is to be
expected. But yeah, more than 2x slower is not nice.
In a quick look at the committed patch, it doesn't seem to have
used any of the speedup strategies we applied to pg_dump a couple
of years ago. One or the other of these should help:
* Issue a single query to fetch stats from every table we're dumping
* Set up a prepared query to avoid re-planning the per-table query
(compare be85727a3)
I'm not sure how workable the first of these would be though.
It's not hard to imagine it blowing out pg_dump's memory usage
for a DB with a lot of tables and high default_statistics_target.
The second one should be relatively downside-free.
regards, tom lane
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:49 ` Re: Statistics Import and Export Tom Lane <[email protected]>
@ 2025-02-21 21:11 ` Andres Freund <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Andres Freund @ 2025-02-21 21:11 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Hi,
On 2025-02-21 15:49:10 -0500, Tom Lane wrote:
> Andres Freund <[email protected]> writes:
> > Which to me rather strongly suggests pg_dump has gotten a *lot* slower with
> > this change.
>
> Well, it's doing strictly more work, so somewhat slower is to be
> expected.
Yea, if we had talked a few percent, I'd not have balked. It's more like 2-4x
though and it'll probably be worse when not connecting over local TCP
connections.
This is a slowdown to the point that the downtime for pg_upgrade will be
substantially lengthened compared to before. But I think we should be able to
address that to a large degree.
> In a quick look at the committed patch, it doesn't seem to have
> used any of the speedup strategies we applied to pg_dump a couple
> of years ago. One or the other of these should help:
>
> * Issue a single query to fetch stats from every table we're dumping
> * Set up a prepared query to avoid re-planning the per-table query
> (compare be85727a3)
>
> I'm not sure how workable the first of these would be though.
> It's not hard to imagine it blowing out pg_dump's memory usage
> for a DB with a lot of tables and high default_statistics_target.
We could presumably do the one-query approach for the relation stats, that's
just three integers. That way we'd at least not end up with two queries for
each table (for pg_class.reltuples etc and for pg_stats).
I guess the memory usage could also be addressed by using COPY, but that's
probably unrealistically complicated.
> The second one should be relatively downside-free.
Yea. And at least with pg_dump running locally that's where a lot of the CPU
time is spent.
Remotely doing lots of one-by-one queries will hurt even with prepared
statements though.
One way to largely address that would be to use a prepared statement combined
with libpq pipelining. That still has separate executor startup etc, but I
think it should still reduce the cost to a point where we don't care anymore.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
@ 2025-02-21 20:57 ` Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Andres Freund @ 2025-02-21 20:57 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Hi,
On 2025-02-21 15:23:00 -0500, Andres Freund wrote:
> On 2025-02-20 01:39:34 -0800, Jeff Davis wrote:
> > Committed with some revisions on top of v48:
>
> This made the pg_upgrade tests considerably slower.
>
> In an assert build without optimization (since that's what I use for normal
> dev work):
>
> 1fd1bd87101^ 65.03s
> 1fd1bd87101 86.84s
>
>
> Looking at the times in the in the regress_log, I see:
> [...]
> Which to me rather strongly suggests pg_dump has gotten a *lot* slower with
> this change.
Indeed. While the slowdown is worse with assertions and without compiler
optimizations, it's pretty bad otherwise too.
optimized, non-cassert, pg_dump and server with the regression database contents:
$ time ./src/bin/pg_dump/pg_dump regression > /dev/null
real 0m1.314s
user 0m0.189s
sys 0m0.059s
$ time ./src/bin/pg_dump/pg_dump --no-statistics regression > /dev/null
real 0m0.472s
user 0m0.179s
sys 0m0.035s
Unoptimized, cassert server and pg_dump:
$ time ./src/bin/pg_dump/pg_dump regression > /dev/null
real 0m9.008s
user 0m0.396s
sys 0m0.108s
$ time ./src/bin/pg_dump/pg_dump --no-statistics regression > /dev/null
real 0m2.590s
user 0m0.347s
sys 0m0.037s
Looking at the query log, the biggest culprit is a *lot* of additional
queries, I think primarily these two:
SELECT c.oid::regclass AS relation, current_setting('server_version_num') AS version, c.relpages, c.reltuples, c.relallvisible FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relname = 'alpha_neg_p2'
SELECT c.oid::regclass AS relation, s.attname,s.inherited,current_setting('server_version_num') AS version, s.null_frac,s.avg_width,s.n_distinct,s.most_common_vals,s.most_common_freqs,s.histogram_bounds,s.correlation,s.most_common_elems,s.most_common_elem_freqs,s.elem_count_histogram,s.range_length_histogram,s.range_empty_frac,s.range_bounds_histogram FROM pg_stats s JOIN pg_namespace n ON n.nspname = s.schemaname JOIN pg_class c ON c.relname = s.tablename AND c.relnamespace = n.oid WHERE s.schemaname = 'public' AND s.tablename = 'alpha_neg_p2' ORDER BY s.attname, s.inherited
I think there are a few things wrong here:
1) Why do we need to plan this over and over? Tom a while ago put in a fair
bit of work to make frequent queries use prepared statements.
In this case we spend more time replanning the query than executing it.
2) Querying this one-by-one makes this much more expensive than if it were
queried in a batched fashion, for multiple tables at once. This is
especially true if actually executed over network, rather than locally.
3) The query is unnecessarily expensive due to repeated joins gathering the
same information. pg_stats has a join to pg_namespace and pg_class, but
then the query above joins to both *again*.
And afaict the joins in the pg_stats query are pretty useless? Isn't all
that information already available in pg_stats? I guess you did that to get
it as a ::regclass, but isn't that already known, why requery it?
4) Why do we need to fetch the version twice for every table, that can't be
right? It won't change while pg_dump is running.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
@ 2025-02-21 21:24 ` Tom Lane <[email protected]>
2025-02-21 22:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
0 siblings, 2 replies; 81+ messages in thread
From: Tom Lane @ 2025-02-21 21:24 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Andres Freund <[email protected]> writes:
> Looking at the query log, the biggest culprit is a *lot* of additional
> queries, I think primarily these two:
> SELECT c.oid::regclass AS relation, current_setting('server_version_num') AS version, c.relpages, c.reltuples, c.relallvisible FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relname = 'alpha_neg_p2'
> SELECT c.oid::regclass AS relation, s.attname,s.inherited,current_setting('server_version_num') AS version, s.null_frac,s.avg_width,s.n_distinct,s.most_common_vals,s.most_common_freqs,s.histogram_bounds,s.correlation,s.most_common_elems,s.most_common_elem_freqs,s.elem_count_histogram,s.range_length_histogram,s.range_empty_frac,s.range_bounds_histogram FROM pg_stats s JOIN pg_namespace n ON n.nspname = s.schemaname JOIN pg_class c ON c.relname = s.tablename AND c.relnamespace = n.oid WHERE s.schemaname = 'public' AND s.tablename = 'alpha_neg_p2' ORDER BY s.attname, s.inherited
Oy. Those are outright horrid, even without any consideration of
pre-preparing them. We know the OID of the table we want to dump,
we should be doing "FROM pg_class WHERE oid = whatever" and lose
the join to pg_namespace altogether. The explicit casts to regclass
are quite expensive too to fetch information that pg_dump already
has. It already knows the server version, too.
Moreover, the first of these shouldn't be a separate query at all.
I objected to fetching pg_statistic content for all tables at once,
but relpages/reltuples/relallvisible is a pretty small amount of
new info. We should just collect those fields as part of getTables'
main query of pg_class (which, indeed, is already fetching relpages).
On the second one, if we want to go through the pg_stats view then
we can't rely on table OID, but I don't see why we need the joins
to anything else. "WHERE s.schemaname = 'x' AND s.tablename = 'y'"
seems sufficient.
I wonder whether we ought to issue different queries depending on
whether we're superuser. The pg_stats view is rather expensive
because of its security restrictions, and if we're superuser we
could just look directly at pg_statistic. Maybe those checks are
fast enough not to matter, but ...
regards, tom lane
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
@ 2025-02-21 22:09 ` Corey Huinker <[email protected]>
2025-02-21 22:20 ` Re: Statistics Import and Export Tom Lane <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-02-21 22:09 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Andres Freund <[email protected]>; Jeff Davis <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
> Oy. Those are outright horrid, even without any consideration of
> pre-preparing them. We know the OID of the table we want to dump,
> we should be doing "FROM pg_class WHERE oid = whatever" and lose
> the join to pg_namespace altogether. The explicit casts to regclass
> are quite expensive too to fetch information that pg_dump already
> has. It already knows the server version, too.
>
+1
Earlier versions had prepared statements, but those were removed to keep
things simple. Easy enough to revive.
>
> Moreover, the first of these shouldn't be a separate query at all.
> I objected to fetching pg_statistic content for all tables at once,
> but relpages/reltuples/relallvisible is a pretty small amount of
> new info. We should just collect those fields as part of getTables'
> main query of pg_class (which, indeed, is already fetching relpages).
>
+1
> On the second one, if we want to go through the pg_stats view then
> we can't rely on table OID, but I don't see why we need the joins
> to anything else. "WHERE s.schemaname = 'x' AND s.tablename = 'y'"
> seems sufficient.
>
+1
>
> I wonder whether we ought to issue different queries depending on
> whether we're superuser. The pg_stats view is rather expensive
> because of its security restrictions, and if we're superuser we
> could just look directly at pg_statistic. Maybe those checks are
> fast enough not to matter, but ...
That could lead to a rather complicated query that has to replicate the
guts of pg_stats for every server-specific version of pg_stats,
specifically the CASE statements that transform
the stakindN/stanumbersN/stavaluesN to mcv, correlation, etc, so I'd like
to avoid that if possible.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-21 22:20 ` Tom Lane <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Tom Lane @ 2025-02-21 22:20 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Andres Freund <[email protected]>; Jeff Davis <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Corey Huinker <[email protected]> writes:
>> I wonder whether we ought to issue different queries depending on
>> whether we're superuser. The pg_stats view is rather expensive
>> because of its security restrictions, and if we're superuser we
>> could just look directly at pg_statistic. Maybe those checks are
>> fast enough not to matter, but ...
> That could lead to a rather complicated query that has to replicate the
> guts of pg_stats for every server-specific version of pg_stats,
> specifically the CASE statements that transform
> the stakindN/stanumbersN/stavaluesN to mcv, correlation, etc, so I'd like
> to avoid that if possible.
Yeah, it'd be notationally ugly for sure. Let's keep that idea in the
back pocket and see how far we get with the other ideas.
regards, tom lane
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
@ 2025-02-21 22:37 ` Andres Freund <[email protected]>
2025-02-21 22:47 ` Re: Statistics Import and Export Tom Lane <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Andres Freund @ 2025-02-21 22:37 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Hi,
On 2025-02-21 16:24:38 -0500, Tom Lane wrote:
> Oy. Those are outright horrid, even without any consideration of
> pre-preparing them. We know the OID of the table we want to dump,
> we should be doing "FROM pg_class WHERE oid = whatever" and lose
> the join to pg_namespace altogether. The explicit casts to regclass
> are quite expensive too to fetch information that pg_dump already
> has. It already knows the server version, too.
> Moreover, the first of these shouldn't be a separate query at all.
> I objected to fetching pg_statistic content for all tables at once,
> but relpages/reltuples/relallvisible is a pretty small amount of
> new info. We should just collect those fields as part of getTables'
> main query of pg_class (which, indeed, is already fetching relpages).
> On the second one, if we want to go through the pg_stats view then
> we can't rely on table OID, but I don't see why we need the joins
> to anything else. "WHERE s.schemaname = 'x' AND s.tablename = 'y'"
> seems sufficient.
Agreed on all those.
> I wonder whether we ought to issue different queries depending on
> whether we're superuser. The pg_stats view is rather expensive
> because of its security restrictions, and if we're superuser we
> could just look directly at pg_statistic. Maybe those checks are
> fast enough not to matter, but ...
It doesn't seem to make much of a difference, from what I can tell.
At execution time most of the time is is in
a) the joins to pg_attribute and pg_class (the ones in pg_stats)
b) array_out().
The times get way worse if you dump stats for catalog tables, because there
some of arrays are regproc and regprocout calls FuncnameGetCandidates(), which
then ends up iterating over a long cached list... I think that's basically
O(N^2)?
Of course that's nothing we should encounter frequently, but ugh.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
@ 2025-02-21 22:47 ` Tom Lane <[email protected]>
2025-02-22 05:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Tom Lane @ 2025-02-21 22:47 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Jeff Davis <[email protected]>; Corey Huinker <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
Andres Freund <[email protected]> writes:
> The times get way worse if you dump stats for catalog tables, because there
> some of arrays are regproc and regprocout calls FuncnameGetCandidates(), which
> then ends up iterating over a long cached list... I think that's basically
> O(N^2)?
Can't be that bad. I don't see any proname values that occur more
than 2 dozen times. You can call that a long list if you want,
but it's not scaling with the size of pg_proc.
> Of course that's nothing we should encounter frequently, but ugh.
Yeah, I can't get excited about the cost of that for normal user
dumps. The 002_pg_dump test does run a dump with --schema pg_catalog,
but it's dubious that that test is worth its cycles.
regards, tom lane
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 22:47 ` Re: Statistics Import and Export Tom Lane <[email protected]>
@ 2025-02-22 05:00 ` Corey Huinker <[email protected]>
2025-02-23 00:03 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-24 00:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 2 replies; 81+ messages in thread
From: Corey Huinker @ 2025-02-22 05:00 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Andres Freund <[email protected]>; Jeff Davis <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
>
>
>
> Yeah, I can't get excited about the cost of that for normal user
> dumps. The 002_pg_dump test does run a dump with --schema pg_catalog,
> but it's dubious that that test is worth its cycles.
>
>
Attached is the first optimization, which gets rid of the pg_class queries
entirely, instead getting the same information from the existing queries in
getTables and getIndexes.
Additionally, the string representation of the server version number is now
stored in the Archive struct. Yes, we already have remoteVersion, but
that's in integer form, and remoteVersionStr is "18devel" rather than
"180000".
I didn't include any work on the attribute query as I wanted to keep that
separate for clarity purposes.
Attachments:
[text/x-patch] v1-0001-Leverage-existing-functions-for-relation-stats.patch (12.3K, ../../CADkLM=f0a43aTd88xW4xCFayEF25g-7hTrHX_WhV40HyocsUGg@mail.gmail.com/3-v1-0001-Leverage-existing-functions-for-relation-stats.patch)
download | inline diff:
From 45467a69813cbf25c2850b254c5d2710c231a723 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Fri, 21 Feb 2025 23:31:04 -0500
Subject: [PATCH v1] Leverage existing functions for relation stats.
Rather than query pg_class once per relation in order to fetch relation
stats (reltuples, relpages, relallvisible), instead just add those
fields to the existing queries in getTables and getIndexes, and then
store their string representations in RelStatsInfo.
Additionally, we will need the string representation of the server
version number so render that one and store in the Archive struct.
No modification has been made for the attribute query, that will be
addressed in a later patch.
---
src/bin/pg_dump/pg_backup.h | 1 +
src/bin/pg_dump/pg_backup_db.c | 1 +
src/bin/pg_dump/pg_dump.c | 113 +++++++++++++--------------------
src/bin/pg_dump/pg_dump.h | 9 +++
4 files changed, 54 insertions(+), 70 deletions(-)
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 350cf659c41..bbca3419b45 100644
--- a/src/bin/pg_dump/pg_backup.h
+++ b/src/bin/pg_dump/pg_backup.h
@@ -224,6 +224,7 @@ typedef struct Archive
int verbose;
char *remoteVersionStr; /* server's version string */
int remoteVersion; /* same in numeric form */
+ char remoteVersionNumStr[32]; /* server version number, as string */
bool isStandby; /* is server a standby node */
int minRemoteVersion; /* allowable range */
diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c
index 71c55d2466a..59c7b70d90f 100644
--- a/src/bin/pg_dump/pg_backup_db.c
+++ b/src/bin/pg_dump/pg_backup_db.c
@@ -41,6 +41,7 @@ _check_database_version(ArchiveHandle *AH)
AH->public.remoteVersionStr = pg_strdup(remoteversion_str);
AH->public.remoteVersion = remoteversion;
+ sprintf(AH->public.remoteVersionNumStr, "%d", remoteversion);
if (!AH->archiveRemoteVersion)
AH->archiveRemoteVersion = AH->public.remoteVersionStr;
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index afd79287177..4311ed5c65d 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -6814,7 +6814,8 @@ getFuncs(Archive *fout)
*
*/
static RelStatsInfo *
-getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+getRelationStatistics(Archive *fout, DumpableObject *rel, const char *relpages,
+ const char *reltuples, const char *relallvisible, char relkind)
{
if (!fout->dopt->dumpStatistics)
return NULL;
@@ -6839,6 +6840,9 @@ getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
dobj->components |= DUMP_COMPONENT_STATISTICS;
dobj->name = pg_strdup(rel->name);
dobj->namespace = rel->namespace;
+ info->relpages = relpages;
+ info->reltuples = reltuples;
+ info->relallvisible = relallvisible;
info->relkind = relkind;
info->postponed_def = false;
@@ -6874,6 +6878,8 @@ getTables(Archive *fout, int *numTables)
int i_relhasindex;
int i_relhasrules;
int i_relpages;
+ int i_reltuples;
+ int i_relallvisible;
int i_toastpages;
int i_owning_tab;
int i_owning_col;
@@ -6921,6 +6927,7 @@ getTables(Archive *fout, int *numTables)
appendPQExpBufferStr(query,
"SELECT c.tableoid, c.oid, c.relname, "
"c.relnamespace, c.relkind, c.reltype, "
+ "c.relpages, c.reltuples, c.relallvisible, "
"c.relowner, "
"c.relchecks, "
"c.relhasindex, c.relhasrules, c.relpages, "
@@ -7088,6 +7095,8 @@ getTables(Archive *fout, int *numTables)
i_relhasindex = PQfnumber(res, "relhasindex");
i_relhasrules = PQfnumber(res, "relhasrules");
i_relpages = PQfnumber(res, "relpages");
+ i_reltuples = PQfnumber(res, "reltuples");
+ i_relallvisible = PQfnumber(res, "relallvisible");
i_toastpages = PQfnumber(res, "toastpages");
i_owning_tab = PQfnumber(res, "owning_tab");
i_owning_col = PQfnumber(res, "owning_col");
@@ -7151,7 +7160,10 @@ getTables(Archive *fout, int *numTables)
tblinfo[i].ncheck = atoi(PQgetvalue(res, i, i_relchecks));
tblinfo[i].hasindex = (strcmp(PQgetvalue(res, i, i_relhasindex), "t") == 0);
tblinfo[i].hasrules = (strcmp(PQgetvalue(res, i, i_relhasrules), "t") == 0);
- tblinfo[i].relpages = atoi(PQgetvalue(res, i, i_relpages));
+ tblinfo[i].relpages_s = pg_strdup(PQgetvalue(res, i, i_relpages));
+ tblinfo[i].relpages = atoi(tblinfo[i].relpages_s);
+ tblinfo[i].reltuples_s = pg_strdup(PQgetvalue(res, i, i_reltuples));
+ tblinfo[i].relallvisible_s = pg_strdup(PQgetvalue(res, i, i_relallvisible));
if (PQgetisnull(res, i, i_toastpages))
tblinfo[i].toastpages = 0;
else
@@ -7233,7 +7245,9 @@ getTables(Archive *fout, int *numTables)
/* Add statistics */
if (tblinfo[i].interesting)
- getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relpages_s,
+ tblinfo[i].reltuples_s, tblinfo[i].relallvisible_s,
+ tblinfo[i].relkind);
/*
* Read-lock target tables to make sure they aren't DROPPED or altered
@@ -7499,6 +7513,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
i_oid,
i_indrelid,
i_indexname,
+ i_relpages,
+ i_reltuples,
+ i_relallvisible,
i_parentidx,
i_indexdef,
i_indnkeyatts,
@@ -7552,6 +7569,7 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
appendPQExpBufferStr(query,
"SELECT t.tableoid, t.oid, i.indrelid, "
"t.relname AS indexname, "
+ "t.relpages, t.reltuples, t.relallvisible, "
"pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, "
"i.indkey, i.indisclustered, "
"c.contype, c.conname, "
@@ -7659,6 +7677,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
i_oid = PQfnumber(res, "oid");
i_indrelid = PQfnumber(res, "indrelid");
i_indexname = PQfnumber(res, "indexname");
+ i_relpages = PQfnumber(res, "relpages");
+ i_reltuples = PQfnumber(res, "reltuples");
+ i_relallvisible = PQfnumber(res, "relallvisible");
i_parentidx = PQfnumber(res, "parentidx");
i_indexdef = PQfnumber(res, "indexdef");
i_indnkeyatts = PQfnumber(res, "indnkeyatts");
@@ -7732,6 +7753,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
AssignDumpId(&indxinfo[j].dobj);
indxinfo[j].dobj.dump = tbinfo->dobj.dump;
indxinfo[j].dobj.name = pg_strdup(PQgetvalue(res, j, i_indexname));
+ indxinfo[j].relpages_s = pg_strdup(PQgetvalue(res, j, i_relpages));
+ indxinfo[j].reltuples_s = pg_strdup(PQgetvalue(res, j, i_reltuples));
+ indxinfo[j].relallvisible_s = pg_strdup(PQgetvalue(res, j, i_relallvisible));
indxinfo[j].dobj.namespace = tbinfo->dobj.namespace;
indxinfo[j].indextable = tbinfo;
indxinfo[j].indexdef = pg_strdup(PQgetvalue(res, j, i_indexdef));
@@ -7759,7 +7783,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
indexkind = RELKIND_PARTITIONED_INDEX;
contype = *(PQgetvalue(res, j, i_contype));
- relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indxinfo[j].relpages_s,
+ indxinfo[j].reltuples_s, indxinfo[j].relallvisible_s,
+ indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -10383,18 +10409,6 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
-/*
- * Tabular description of the parameters to pg_restore_relation_stats()
- * param_name, param_type
- */
-static const char *rel_stats_arginfo[][2] = {
- {"relation", "regclass"},
- {"version", "integer"},
- {"relpages", "integer"},
- {"reltuples", "real"},
- {"relallvisible", "integer"},
-};
-
/*
* Tabular description of the parameters to pg_restore_attribute_stats()
* param_name, param_type
@@ -10419,30 +10433,6 @@ static const char *att_stats_arginfo[][2] = {
{"range_bounds_histogram", "text"},
};
-/*
- * getRelStatsExportQuery --
- *
- * Generate a query that will fetch all relation (e.g. pg_class)
- * stats for a given relation.
- */
-static void
-getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
- const char *schemaname, const char *relname)
-{
- resetPQExpBuffer(query);
- appendPQExpBufferStr(query,
- "SELECT c.oid::regclass AS relation, "
- "current_setting('server_version_num') AS version, "
- "c.relpages, c.reltuples, c.relallvisible "
- "FROM pg_class c "
- "JOIN pg_namespace n "
- "ON n.oid = c.relnamespace "
- "WHERE n.nspname = ");
- appendStringLiteralAH(query, schemaname, fout);
- appendPQExpBufferStr(query, " AND c.relname = ");
- appendStringLiteralAH(query, relname, fout);
-}
-
/*
* getAttStatsExportQuery --
*
@@ -10521,33 +10511,20 @@ appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
* Append a formatted pg_restore_relation_stats statement.
*/
static void
-appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+appendRelStatsImport(PQExpBuffer out, Archive *fout, const RelStatsInfo *rsinfo)
{
- const char *sep = "";
-
- if (PQntuples(res) == 0)
- return;
+ const char *qualname = fmtQualifiedId(rsinfo->dobj.namespace->dobj.name, rsinfo->dobj.name);
appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
-
- for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
- {
- const char *argname = rel_stats_arginfo[argno][0];
- const char *argtype = rel_stats_arginfo[argno][1];
- int fieldno = PQfnumber(res, argname);
-
- if (fieldno < 0)
- pg_fatal("relation stats export query missing field '%s'",
- argname);
-
- if (PQgetisnull(res, 0, fieldno))
- continue;
-
- appendPQExpBufferStr(out, sep);
- appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
-
- sep = ",\n";
- }
+ appendNamedArgument(out, fout, "relation", qualname, "regclass");
+ appendPQExpBufferStr(out, ",\n");
+ appendNamedArgument(out, fout, "version", fout->remoteVersionNumStr, "integer");
+ appendPQExpBufferStr(out, ",\n");
+ appendNamedArgument(out, fout, "relpages", rsinfo->relpages, "integer");
+ appendPQExpBufferStr(out, ",\n");
+ appendNamedArgument(out, fout, "reltuples", rsinfo->reltuples, "real");
+ appendPQExpBufferStr(out, ",\n");
+ appendNamedArgument(out, fout, "relallvisible", rsinfo->relallvisible, "integer");
appendPQExpBufferStr(out, "\n);\n");
}
@@ -10643,15 +10620,11 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
tag = createPQExpBuffer();
appendPQExpBufferStr(tag, fmtId(dobj->name));
- query = createPQExpBuffer();
out = createPQExpBuffer();
- getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
- dobj->name);
- res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
- appendRelStatsImport(out, fout, res);
- PQclear(res);
+ appendRelStatsImport(out, fout, rsinfo);
+ query = createPQExpBuffer();
getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
dobj->name);
res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index f08f5905aa3..1fd1812d348 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -329,6 +329,9 @@ typedef struct _tableInfo
int owning_col; /* attr # of column owning sequence */
bool is_identity_sequence;
int relpages; /* table's size in pages (from pg_class) */
+ const char *relpages_s; /* table's size in pages (from pg_class) */
+ const char *reltuples_s; /* table's approx number of tuples (from pg_class) */
+ const char *relallvisible_s; /* table's number of pages all-visible (from pg_class) */
int toastpages; /* toast table's size in pages, if any */
bool interesting; /* true if need to collect more data */
@@ -418,6 +421,9 @@ typedef struct _indxInfo
int indnattrs; /* total number of index attributes */
Oid *indkeys; /* In spite of the name 'indkeys' this field
* contains both key and nonkey attributes */
+ const char *relpages_s;
+ const char *reltuples_s;
+ const char *relallvisible_s;
bool indisclustered;
bool indisreplident;
bool indnullsnotdistinct;
@@ -438,6 +444,9 @@ typedef struct _indexAttachInfo
typedef struct _relStatsInfo
{
DumpableObject dobj;
+ const char *relpages;
+ const char *reltuples;
+ const char *relallvisible;
char relkind; /* 'r', 'm', 'i', etc */
bool postponed_def; /* stats must be postponed into post-data */
} RelStatsInfo;
base-commit: f8d7f29b3e81db59b95e4b5baaa6943178c89fd8
--
2.48.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 22:47 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-22 05:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-23 00:03 ` Tom Lane <[email protected]>
1 sibling, 0 replies; 81+ messages in thread
From: Tom Lane @ 2025-02-23 00:03 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Andres Freund <[email protected]>; Jeff Davis <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
BTW, while nosing around looking for an explanation for our
cross-version-upgrade woes, I chanced to notice that
relation_statistics_update rejects negative values for
relpages and relallvisible. This is nonsense. Internally
those values are BlockNumbers and can have any value from
0 to UINT32_MAX. We represent them as signed int32 at the
SQL level, which means they can read out as any int32 value.
So the range checks that are being applied to them are flat
wrong and should be removed. Admittedly, you'd need a table
exceeding 16TB (if I did the math right) to see a problem,
but that doesn't make it not wrong.
It might be a good idea to change the code so that it declares
these values internally as BlockNumber and uses PG_GETARG_UINT32,
but I think that would only be a cosmetic change not a
correctness issue.
regards, tom lane
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 22:47 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-22 05:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-24 00:21 ` Jeff Davis <[email protected]>
2025-02-24 01:03 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
1 sibling, 1 reply; 81+ messages in thread
From: Jeff Davis @ 2025-02-24 00:21 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; Tom Lane <[email protected]>; +Cc: Andres Freund <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Sat, 2025-02-22 at 00:00 -0500, Corey Huinker wrote:
>
> Attached is the first optimization, which gets rid of the pg_class
> queries entirely, instead getting the same information from the
> existing queries in getTables and getIndexes.
Attached a revised version. The main changes are that the only struct
it changes is RelStatsInfo, and it doesn't carry around string values.
IIUC, your version carried around the string values so that there would
be no conversion; it would hold the string from one result to the next.
That makes sense, but it seemed to change a lot of struct fields, and
have unnecessary string copying and memory usage which was not freed.
Instead, I used float_to_shortest_decimal_buf(), which is what
float4out() uses, which should be a consistent way to convert the float
value.
That meant that we couldn't use appendNamedArgument() as easily, but it
wasn't helping much in that function anyway, because it was no longer a
loop.
I didn't measure any performance difference between your version and
mine, but avoiding a few allocations couldn't hurt. It seems to save
just under 20% on an unoptimized build.
Regards,
Jeff Davis
Attachments:
[text/x-patch] v2j-0001-Avoid-unnecessary-relation-stats-query-in-pg_dum.patch (11.3K, ../../[email protected]/2-v2j-0001-Avoid-unnecessary-relation-stats-query-in-pg_dum.patch)
download | inline diff:
From 2f16b7cf941fc14e3156e8ddc536c8f93f97eb77 Mon Sep 17 00:00:00 2001
From: Corey Huinker <[email protected]>
Date: Fri, 21 Feb 2025 23:31:04 -0500
Subject: [PATCH v2j] Avoid unnecessary relation stats query in pg_dump.
The few fields we need can be easily collected in getTables() and
getIndexes() and stored in RelStatsInfo.
Co-authored-by: Corey Huinker <[email protected]>
Co-authored-by: Jeff Davis <[email protected]>
Discussion: https://postgr.es/m/CADkLM=f0a43aTd88xW4xCFayEF25g-7hTrHX_WhV40HyocsUGg@mail.gmail.com
---
src/bin/pg_dump/pg_dump.c | 142 +++++++++++++++-----------------------
src/bin/pg_dump/pg_dump.h | 5 +-
2 files changed, 61 insertions(+), 86 deletions(-)
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index afd79287177..d119ce716b0 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -56,6 +56,7 @@
#include "common/connect.h"
#include "common/int.h"
#include "common/relpath.h"
+#include "common/shortest_dec.h"
#include "compress_io.h"
#include "dumputils.h"
#include "fe_utils/option_utils.h"
@@ -6814,7 +6815,8 @@ getFuncs(Archive *fout)
*
*/
static RelStatsInfo *
-getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
+getRelationStatistics(Archive *fout, DumpableObject *rel, int32 relpages,
+ float reltuples, int32 relallvisible, char relkind)
{
if (!fout->dopt->dumpStatistics)
return NULL;
@@ -6839,6 +6841,9 @@ getRelationStatistics(Archive *fout, DumpableObject *rel, char relkind)
dobj->components |= DUMP_COMPONENT_STATISTICS;
dobj->name = pg_strdup(rel->name);
dobj->namespace = rel->namespace;
+ info->relpages = relpages;
+ info->reltuples = reltuples;
+ info->relallvisible = relallvisible;
info->relkind = relkind;
info->postponed_def = false;
@@ -6874,6 +6879,8 @@ getTables(Archive *fout, int *numTables)
int i_relhasindex;
int i_relhasrules;
int i_relpages;
+ int i_reltuples;
+ int i_relallvisible;
int i_toastpages;
int i_owning_tab;
int i_owning_col;
@@ -6924,7 +6931,7 @@ getTables(Archive *fout, int *numTables)
"c.relowner, "
"c.relchecks, "
"c.relhasindex, c.relhasrules, c.relpages, "
- "c.relhastriggers, "
+ "c.reltuples, c.relallvisible, c.relhastriggers, "
"c.relpersistence, "
"c.reloftype, "
"c.relacl, "
@@ -7088,6 +7095,8 @@ getTables(Archive *fout, int *numTables)
i_relhasindex = PQfnumber(res, "relhasindex");
i_relhasrules = PQfnumber(res, "relhasrules");
i_relpages = PQfnumber(res, "relpages");
+ i_reltuples = PQfnumber(res, "reltuples");
+ i_relallvisible = PQfnumber(res, "relallvisible");
i_toastpages = PQfnumber(res, "toastpages");
i_owning_tab = PQfnumber(res, "owning_tab");
i_owning_col = PQfnumber(res, "owning_col");
@@ -7134,6 +7143,9 @@ getTables(Archive *fout, int *numTables)
for (i = 0; i < ntups; i++)
{
+ float reltuples = atof(PQgetvalue(res, i, i_reltuples));
+ int32 relallvisible = atoi(PQgetvalue(res, i, i_relallvisible));
+
tblinfo[i].dobj.objType = DO_TABLE;
tblinfo[i].dobj.catId.tableoid = atooid(PQgetvalue(res, i, i_reltableoid));
tblinfo[i].dobj.catId.oid = atooid(PQgetvalue(res, i, i_reloid));
@@ -7233,7 +7245,8 @@ getTables(Archive *fout, int *numTables)
/* Add statistics */
if (tblinfo[i].interesting)
- getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relkind);
+ getRelationStatistics(fout, &tblinfo[i].dobj, tblinfo[i].relpages,
+ reltuples, relallvisible, tblinfo[i].relkind);
/*
* Read-lock target tables to make sure they aren't DROPPED or altered
@@ -7499,6 +7512,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
i_oid,
i_indrelid,
i_indexname,
+ i_relpages,
+ i_reltuples,
+ i_relallvisible,
i_parentidx,
i_indexdef,
i_indnkeyatts,
@@ -7552,6 +7568,7 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
appendPQExpBufferStr(query,
"SELECT t.tableoid, t.oid, i.indrelid, "
"t.relname AS indexname, "
+ "t.relpages, t.reltuples, t.relallvisible, "
"pg_catalog.pg_get_indexdef(i.indexrelid) AS indexdef, "
"i.indkey, i.indisclustered, "
"c.contype, c.conname, "
@@ -7659,6 +7676,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
i_oid = PQfnumber(res, "oid");
i_indrelid = PQfnumber(res, "indrelid");
i_indexname = PQfnumber(res, "indexname");
+ i_relpages = PQfnumber(res, "relpages");
+ i_reltuples = PQfnumber(res, "reltuples");
+ i_relallvisible = PQfnumber(res, "relallvisible");
i_parentidx = PQfnumber(res, "parentidx");
i_indexdef = PQfnumber(res, "indexdef");
i_indnkeyatts = PQfnumber(res, "indnkeyatts");
@@ -7725,6 +7745,9 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
char contype;
char indexkind;
RelStatsInfo *relstats;
+ int32 relpages = atoi(PQgetvalue(res, j, i_relpages));
+ float reltuples = atof(PQgetvalue(res, j, i_reltuples));
+ int32 relallvisible = atoi(PQgetvalue(res, j, i_relallvisible));
indxinfo[j].dobj.objType = DO_INDEX;
indxinfo[j].dobj.catId.tableoid = atooid(PQgetvalue(res, j, i_tableoid));
@@ -7759,7 +7782,8 @@ getIndexes(Archive *fout, TableInfo tblinfo[], int numTables)
indexkind = RELKIND_PARTITIONED_INDEX;
contype = *(PQgetvalue(res, j, i_contype));
- relstats = getRelationStatistics(fout, &indxinfo[j].dobj, indexkind);
+ relstats = getRelationStatistics(fout, &indxinfo[j].dobj, relpages,
+ reltuples, relallvisible, indexkind);
if (contype == 'p' || contype == 'u' || contype == 'x')
{
@@ -10383,18 +10407,6 @@ dumpComment(Archive *fout, const char *type,
catalogId, subid, dumpId, NULL);
}
-/*
- * Tabular description of the parameters to pg_restore_relation_stats()
- * param_name, param_type
- */
-static const char *rel_stats_arginfo[][2] = {
- {"relation", "regclass"},
- {"version", "integer"},
- {"relpages", "integer"},
- {"reltuples", "real"},
- {"relallvisible", "integer"},
-};
-
/*
* Tabular description of the parameters to pg_restore_attribute_stats()
* param_name, param_type
@@ -10419,30 +10431,6 @@ static const char *att_stats_arginfo[][2] = {
{"range_bounds_histogram", "text"},
};
-/*
- * getRelStatsExportQuery --
- *
- * Generate a query that will fetch all relation (e.g. pg_class)
- * stats for a given relation.
- */
-static void
-getRelStatsExportQuery(PQExpBuffer query, Archive *fout,
- const char *schemaname, const char *relname)
-{
- resetPQExpBuffer(query);
- appendPQExpBufferStr(query,
- "SELECT c.oid::regclass AS relation, "
- "current_setting('server_version_num') AS version, "
- "c.relpages, c.reltuples, c.relallvisible "
- "FROM pg_class c "
- "JOIN pg_namespace n "
- "ON n.oid = c.relnamespace "
- "WHERE n.nspname = ");
- appendStringLiteralAH(query, schemaname, fout);
- appendPQExpBufferStr(query, " AND c.relname = ");
- appendStringLiteralAH(query, relname, fout);
-}
-
/*
* getAttStatsExportQuery --
*
@@ -10454,21 +10442,22 @@ getAttStatsExportQuery(PQExpBuffer query, Archive *fout,
const char *schemaname, const char *relname)
{
resetPQExpBuffer(query);
- appendPQExpBufferStr(query,
- "SELECT c.oid::regclass AS relation, "
- "s.attname,"
- "s.inherited,"
- "current_setting('server_version_num') AS version, "
- "s.null_frac,"
- "s.avg_width,"
- "s.n_distinct,"
- "s.most_common_vals,"
- "s.most_common_freqs,"
- "s.histogram_bounds,"
- "s.correlation,"
- "s.most_common_elems,"
- "s.most_common_elem_freqs,"
- "s.elem_count_histogram,");
+ appendPQExpBuffer(query,
+ "SELECT c.oid::regclass AS relation, "
+ "s.attname,"
+ "s.inherited,"
+ "'%u'::integer AS version, "
+ "s.null_frac,"
+ "s.avg_width,"
+ "s.n_distinct,"
+ "s.most_common_vals,"
+ "s.most_common_freqs,"
+ "s.histogram_bounds,"
+ "s.correlation,"
+ "s.most_common_elems,"
+ "s.most_common_elem_freqs,"
+ "s.elem_count_histogram,",
+ fout->remoteVersion);
if (fout->remoteVersion >= 170000)
appendPQExpBufferStr(query,
@@ -10521,34 +10510,21 @@ appendNamedArgument(PQExpBuffer out, Archive *fout, const char *argname,
* Append a formatted pg_restore_relation_stats statement.
*/
static void
-appendRelStatsImport(PQExpBuffer out, Archive *fout, PGresult *res)
+appendRelStatsImport(PQExpBuffer out, Archive *fout, const RelStatsInfo *rsinfo)
{
- const char *sep = "";
+ const char *qualname = fmtQualifiedId(rsinfo->dobj.namespace->dobj.name, rsinfo->dobj.name);
+ char reltuples_str[FLOAT_SHORTEST_DECIMAL_LEN];
- if (PQntuples(res) == 0)
- return;
+ float_to_shortest_decimal_buf(rsinfo->reltuples, reltuples_str);
appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_relation_stats(\n");
-
- for (int argno = 0; argno < lengthof(rel_stats_arginfo); argno++)
- {
- const char *argname = rel_stats_arginfo[argno][0];
- const char *argtype = rel_stats_arginfo[argno][1];
- int fieldno = PQfnumber(res, argname);
-
- if (fieldno < 0)
- pg_fatal("relation stats export query missing field '%s'",
- argname);
-
- if (PQgetisnull(res, 0, fieldno))
- continue;
-
- appendPQExpBufferStr(out, sep);
- appendNamedArgument(out, fout, argname, PQgetvalue(res, 0, fieldno), argtype);
-
- sep = ",\n";
- }
- appendPQExpBufferStr(out, "\n);\n");
+ appendPQExpBuffer(out, "\t'relation', '%s'::regclass,\n", qualname);
+ appendPQExpBuffer(out, "\t'version', '%u'::integer,\n",
+ fout->remoteVersion);
+ appendPQExpBuffer(out, "\t'relpages', '%d'::integer,\n", rsinfo->relpages);
+ appendPQExpBuffer(out, "\t'reltuples', '%s'::real,\n", reltuples_str);
+ appendPQExpBuffer(out, "\t'relallvisible', '%d'::integer\n);\n",
+ rsinfo->relallvisible);
}
/*
@@ -10643,15 +10619,11 @@ dumpRelationStats(Archive *fout, const RelStatsInfo *rsinfo)
tag = createPQExpBuffer();
appendPQExpBufferStr(tag, fmtId(dobj->name));
- query = createPQExpBuffer();
out = createPQExpBuffer();
- getRelStatsExportQuery(query, fout, dobj->namespace->dobj.name,
- dobj->name);
- res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
- appendRelStatsImport(out, fout, res);
- PQclear(res);
+ appendRelStatsImport(out, fout, rsinfo);
+ query = createPQExpBuffer();
getAttStatsExportQuery(query, fout, dobj->namespace->dobj.name,
dobj->name);
res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index f08f5905aa3..9d6a4857c4b 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -328,7 +328,7 @@ typedef struct _tableInfo
Oid owning_tab; /* OID of table owning sequence */
int owning_col; /* attr # of column owning sequence */
bool is_identity_sequence;
- int relpages; /* table's size in pages (from pg_class) */
+ int32 relpages; /* table's size in pages (from pg_class) */
int toastpages; /* toast table's size in pages, if any */
bool interesting; /* true if need to collect more data */
@@ -438,6 +438,9 @@ typedef struct _indexAttachInfo
typedef struct _relStatsInfo
{
DumpableObject dobj;
+ int32 relpages;
+ float reltuples;
+ int32 relallvisible;
char relkind; /* 'r', 'm', 'i', etc */
bool postponed_def; /* stats must be postponed into post-data */
} RelStatsInfo;
--
2.34.1
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 22:47 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-22 05:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-24 00:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
@ 2025-02-24 01:03 ` Corey Huinker <[email protected]>
2025-02-24 06:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
0 siblings, 1 reply; 81+ messages in thread
From: Corey Huinker @ 2025-02-24 01:03 UTC (permalink / raw)
To: Jeff Davis <[email protected]>; +Cc: Tom Lane <[email protected]>; Andres Freund <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Sun, Feb 23, 2025 at 7:22 PM Jeff Davis <[email protected]> wrote:
> On Sat, 2025-02-22 at 00:00 -0500, Corey Huinker wrote:
> >
> > Attached is the first optimization, which gets rid of the pg_class
> > queries entirely, instead getting the same information from the
> > existing queries in getTables and getIndexes.
>
> Attached a revised version. The main changes are that the only struct
> it changes is RelStatsInfo, and it doesn't carry around string values.
>
> IIUC, your version carried around the string values so that there would
> be no conversion; it would hold the string from one result to the next.
> That makes sense, but it seemed to change a lot of struct fields, and
> have unnecessary string copying and memory usage which was not freed.
> Instead, I used float_to_shortest_decimal_buf(), which is what
> float4out() uses, which should be a consistent way to convert the float
> value.
>
If we're fine with giving up on appendNamedArgument() for relstats,
wouldn't we also want to mash these into a single call?
appendPQExpBuffer(out, "\t'relation', '%s'::regclass,\n", qualname);
appendPQExpBuffer(out, "\t'version', '%u'::integer,\n",
fout->remoteVersion);
appendPQExpBuffer(out, "\t'relpages', '%d'::integer,\n", rsinfo->relpages);
appendPQExpBuffer(out, "\t'reltuples', '%s'::real,\n", reltuples_str);
appendPQExpBuffer(out, "\t'relallvisible', '%d'::integer\n);\n",
rsinfo->relallvisible);
to:
appendPQExpBuffer(out, "\t'relation', '%s'::regclass"
",\n\t'version', '%u'::integer"
",\n\t'relpages', '%d'::integer"
",\n\t'reltuples', '%s'::real"
",\n\t'relallvisible', '%d'::integer",
qualname, fout->remoteVersion, rsinfo->relpages,
rsinfo->reltuples_str, rsinfo->relallvisible);
appendPQExpBufferStr(out, "\n);\n");
Also, there's work elsewhere that may add relallfrozen to pg_class, which
would be something we'd want to add depending on the remoteVersion, and
this format will make that change pretty clear.
>
> That meant that we couldn't use appendNamedArgument() as easily, but it
> wasn't helping much in that function anyway, because it was no longer a
> loop.
>
It still served to encapsulate the format of a kwarg pair, but little more,
agreed.
>
> I didn't measure any performance difference between your version and
> mine, but avoiding a few allocations couldn't hurt. It seems to save
> just under 20% on an unoptimized build.
>
Part of me thinks we'd want to do the reverse - change the struct to store
char[32] to for each of relpages, reltuples, and relallvisible, and then
convert reltpages to int in the one place where we actually need to use in
its numeric form, and even then only in one place. Conversions to and from
other data types introduce the possibility, though very remote, of the
converted-and-then-unconverted value being cosmetically different from what
we got from the server, and if down the road we're dealing with more
complex data types, those conversions might become significant.
^ permalink raw reply [nested|flat] 81+ messages in thread
* Re: Statistics Import and Export
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 22:47 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-22 05:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-24 00:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-24 01:03 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
@ 2025-02-24 06:39 ` Jeff Davis <[email protected]>
0 siblings, 0 replies; 81+ messages in thread
From: Jeff Davis @ 2025-02-24 06:39 UTC (permalink / raw)
To: Corey Huinker <[email protected]>; +Cc: Tom Lane <[email protected]>; Andres Freund <[email protected]>; Michael Paquier <[email protected]>; jian he <[email protected]>; Nathan Bossart <[email protected]>; Bruce Momjian <[email protected]>; Matthias van de Meent <[email protected]>; Magnus Hagander <[email protected]>; Stephen Frost <[email protected]>; Ashutosh Bapat <[email protected]>; Peter Smith <[email protected]>; PostgreSQL Hackers <[email protected]>; [email protected]
On Sun, 2025-02-23 at 20:03 -0500, Corey Huinker wrote:
> If we're fine with giving up on appendNamedArgument() for relstats,
> wouldn't we also want to mash these into a single call?
...
> appendPQExpBuffer(out, "\t'relation', '%s'::regclass"
> ",\n\t'version', '%u'::integer"
> ",\n\t'relpages', '%d'::integer"
> ",\n\t'reltuples', '%s'::real"
> ",\n\t'relallvisible', '%d'::integer",
> qualname, fout->remoteVersion, rsinfo-
> >relpages,
> rsinfo->reltuples_str, rsinfo->relallvisible);
> appendPQExpBufferStr(out, "\n);\n");
+1.
>
>
> Part of me thinks we'd want to do the reverse - change the struct to
> store char[32] to for each of relpages, reltuples, and relallvisible,
> and then convert reltpages to int in the one place where we actually
> need to use in its numeric form, and even then only in one place.
> Conversions to and from other data types introduce the possibility,
> though very remote, of the converted-and-then-unconverted value being
> cosmetically different from what we got from the server, and if down
> the road we're dealing with more complex data types, those
> conversions might become significant.
That's a good point but let's avoid excessive redundancy in the
structures. Adding a few fields to RelStatsInfo should be enough.
Regards,
Jeff Davis
^ permalink raw reply [nested|flat] 81+ messages in thread
end of thread, other threads:[~2025-02-24 06:39 UTC | newest]
Thread overview: 81+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2023-06-25 11:48 [PATCH v1 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-06-26 08:05 [PATCH v2 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-07-26 10:49 [PATCH v3 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-08-09 07:56 [PATCH v4 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-09-02 06:32 [PATCH v5 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-09-12 05:22 [PATCH v6 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-09-22 04:53 [PATCH v7 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-09-25 05:01 [PATCH v8 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-10-04 05:51 [PATCH v9 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-10-22 02:22 [PATCH v10 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-11-08 06:57 [PATCH v11 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2023-12-04 11:23 [PATCH v12 1/7] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-01-22 09:45 [PATCH v13 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-02-28 13:59 [PATCH v14 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-03-28 10:30 [PATCH v15 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-04-12 06:49 [PATCH v16 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-04-28 11:00 [PATCH v17 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-05-11 07:11 [PATCH v18 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-05-14 23:26 [PATCH v19 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-05-24 02:26 [PATCH v20 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-08-26 04:32 [PATCH v21 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-09-19 04:48 [PATCH v22 1/8] Row pattern recognition patch for raw parser. Tatsuo Ishii <[email protected]>
2024-11-26 22:11 Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-11-27 05:08 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 00:44 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-07 19:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-07 21:27 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-12 03:49 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-13 05:22 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-20 05:23 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-20 06:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-21 00:16 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-21 10:11 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2024-12-26 18:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-26 18:54 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2024-12-30 20:02 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2024-12-30 20:45 ` Re: Statistics Import and Export Bruce Momjian <[email protected]>
2025-01-06 21:27 ` Re: Statistics Import and Export Nathan Bossart <[email protected]>
2025-01-07 06:18 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-07 20:14 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-15 20:56 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-16 14:43 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-16 15:52 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-17 14:20 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-18 17:00 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 08:21 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-20 15:58 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 19:11 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-20 21:45 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-01-20 23:31 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-01-27 14:05 ` Re: Statistics Import and Export jian he <[email protected]>
2025-01-27 16:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:01 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-06 04:45 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-06 09:35 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-07 06:07 ` Re: Statistics Import and Export Michael Paquier <[email protected]>
2025-02-10 03:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-10 23:36 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 01:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-11 19:02 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-13 03:00 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 09:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-20 10:24 ` Re: Statistics Import and Export vignesh C <[email protected]>
2025-02-20 15:43 ` Re: Statistics Import and Export Andrew Dunstan <[email protected]>
2025-02-21 07:24 ` RE: Statistics Import and Export Hayato Kuroda (Fujitsu) <[email protected]>
2025-02-22 18:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-21 20:23 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:49 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 21:11 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 20:57 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 21:24 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:09 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-21 22:20 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-21 22:37 ` Re: Statistics Import and Export Andres Freund <[email protected]>
2025-02-21 22:47 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-22 05:00 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-23 00:03 ` Re: Statistics Import and Export Tom Lane <[email protected]>
2025-02-24 00:21 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
2025-02-24 01:03 ` Re: Statistics Import and Export Corey Huinker <[email protected]>
2025-02-24 06:39 ` Re: Statistics Import and Export Jeff Davis <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox