public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v5 2/2] Add a --outdated option to reindexdb
7+ messages / 5 participants
[nested] [flat]
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v2] WIP: Evaluate arguments of correlated SubPlans in the referencing ExprState
@ 2023-02-25 21:39 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2023-02-25 21:39 UTC (permalink / raw)
---
src/include/executor/execExpr.h | 6 +-
src/include/nodes/execnodes.h | 1 -
src/backend/executor/execExpr.c | 93 +++++++++++++++++----------
src/backend/executor/execExprInterp.c | 22 +++++++
src/backend/executor/execProcnode.c | 5 ++
src/backend/executor/nodeSubplan.c | 30 ++++-----
src/backend/jit/llvm/llvmjit_expr.c | 6 ++
src/backend/jit/llvm/llvmjit_types.c | 1 +
8 files changed, 112 insertions(+), 52 deletions(-)
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index 06c3adc0a19..ca2b7306cd0 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -158,6 +158,8 @@ typedef enum ExprEvalOp
EEOP_PARAM_EXEC,
EEOP_PARAM_EXTERN,
EEOP_PARAM_CALLBACK,
+ /* set PARAM_EXEC value */
+ EEOP_PARAM_SET,
/* return CaseTestExpr value */
EEOP_CASE_TESTVAL,
@@ -374,7 +376,7 @@ typedef struct ExprEvalStep
ExprEvalRowtypeCache rowcache;
} nulltest_row;
- /* for EEOP_PARAM_EXEC/EXTERN */
+ /* for EEOP_PARAM_EXEC/EXTERN and EEOP_PARAM_SET */
struct
{
int paramid; /* numeric ID for parameter */
@@ -738,6 +740,8 @@ extern void ExecEvalParamExec(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalParamExtern(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
+extern void ExecEvalParamSet(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
extern void ExecEvalCurrentOfExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalNextValueExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalRowNull(ExprState *state, ExprEvalStep *op,
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index e7eb1e666ff..16e95e4cb48 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -947,7 +947,6 @@ typedef struct SubPlanState
struct PlanState *planstate; /* subselect plan's state tree */
struct PlanState *parent; /* parent plan node's state tree */
ExprState *testexpr; /* state of combining expression */
- List *args; /* states of argument expression(s) */
HeapTuple curTuple; /* copy of most recent tuple from subplan */
Datum curArray; /* most recent array from ARRAY() subplan */
/* these are used when hashing the subselect's output: */
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index c61f23c6c18..002f2a0091f 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -87,6 +87,9 @@ static void ExecBuildAggTransCall(ExprState *state, AggState *aggstate,
FunctionCallInfo fcinfo, AggStatePerTrans pertrans,
int transno, int setno, int setoff, bool ishash,
bool nullcheck);
+static void ExecInitSubPlanExpr(SubPlan *subplan,
+ ExprState *state,
+ Datum *resv, bool *resnull);
/*
@@ -1388,7 +1391,6 @@ ExecInitExprRec(Expr *node, ExprState *state,
case T_SubPlan:
{
SubPlan *subplan = (SubPlan *) node;
- SubPlanState *sstate;
/*
* Real execution of a MULTIEXPR SubPlan has already been
@@ -1405,19 +1407,7 @@ ExecInitExprRec(Expr *node, ExprState *state,
break;
}
- if (!state->parent)
- elog(ERROR, "SubPlan found with no parent plan");
-
- sstate = ExecInitSubPlan(subplan, state->parent);
-
- /* add SubPlanState nodes to state->parent->subPlan */
- state->parent->subPlan = lappend(state->parent->subPlan,
- sstate);
-
- scratch.opcode = EEOP_SUBPLAN;
- scratch.d.subplan.sstate = sstate;
-
- ExprEvalPushStep(state, &scratch);
+ ExecInitSubPlanExpr(subplan, state, resv, resnull);
break;
}
@@ -2618,29 +2608,12 @@ ExecPushExprSetupSteps(ExprState *state, ExprSetupInfo *info)
foreach(lc, info->multiexpr_subplans)
{
SubPlan *subplan = (SubPlan *) lfirst(lc);
- SubPlanState *sstate;
Assert(subplan->subLinkType == MULTIEXPR_SUBLINK);
- /* This should match what ExecInitExprRec does for other SubPlans: */
-
- if (!state->parent)
- elog(ERROR, "SubPlan found with no parent plan");
-
- sstate = ExecInitSubPlan(subplan, state->parent);
-
- /* add SubPlanState nodes to state->parent->subPlan */
- state->parent->subPlan = lappend(state->parent->subPlan,
- sstate);
-
- scratch.opcode = EEOP_SUBPLAN;
- scratch.d.subplan.sstate = sstate;
-
/* The result can be ignored, but we better put it somewhere */
- scratch.resvalue = &state->resvalue;
- scratch.resnull = &state->resnull;
-
- ExprEvalPushStep(state, &scratch);
+ ExecInitSubPlanExpr(subplan, state,
+ &state->resvalue, &state->resnull);
}
}
@@ -4040,3 +4013,57 @@ ExecBuildParamSetEqual(TupleDesc desc,
return state;
}
+
+static void
+ExecInitSubPlanExpr(SubPlan *subplan,
+ ExprState *state,
+ Datum *resv, bool *resnull)
+{
+ ExprEvalStep scratch = {0};
+ SubPlanState *sstate;
+ ListCell *pvar;
+ ListCell *l;
+
+ if (!state->parent)
+ elog(ERROR, "SubPlan found with no parent plan");
+
+ /*
+ * Generate steps to evaluate input arguments for the subplan.
+ *
+ * We evaluate the argument expressions into ExprState's resvalue/resnull,
+ * and then use PARAM_SET to update the parameter. We do that, instead of
+ * evaluating directly into the param, to avoid depending on the pointer
+ * value remaining stable / being included in the generated expression. No
+ * danger of conflicts with other uses of resvalue/resnull as storing and
+ * using the value always is in subsequent steps.
+ *
+ * Any calculation we have to do can be done in the parent econtext, since
+ * the Param values don't need to have per-query lifetime.
+ */
+ forboth(l, subplan->parParam, pvar, subplan->args)
+ {
+ int paramid = lfirst_int(l);
+
+ ExecInitExprRec(lfirst(pvar), state,
+ &state->resvalue, &state->resnull);
+
+ scratch.opcode = EEOP_PARAM_SET;
+ scratch.d.param.paramid = paramid;
+ /* type isn't needed, but an old value could be confusing */
+ scratch.d.param.paramtype = InvalidOid;
+ ExprEvalPushStep(state, &scratch);
+ }
+
+ sstate = ExecInitSubPlan(subplan, state->parent);
+
+ /* add SubPlanState nodes to state->parent->subPlan */
+ state->parent->subPlan = lappend(state->parent->subPlan,
+ sstate);
+
+ scratch.opcode = EEOP_SUBPLAN;
+ scratch.resvalue = resv;
+ scratch.resnull = resnull;
+ scratch.d.subplan.sstate = sstate;
+
+ ExprEvalPushStep(state, &scratch);
+}
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 19351fe34bf..3cab8a5cdae 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -446,6 +446,7 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_PARAM_EXEC,
&&CASE_EEOP_PARAM_EXTERN,
&&CASE_EEOP_PARAM_CALLBACK,
+ &&CASE_EEOP_PARAM_SET,
&&CASE_EEOP_CASE_TESTVAL,
&&CASE_EEOP_MAKE_READONLY,
&&CASE_EEOP_IOCOERCE,
@@ -1081,6 +1082,13 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_PARAM_SET)
+ {
+ /* out of line, unlikely to matter performancewise */
+ ExecEvalParamSet(state, op, econtext);
+ EEO_NEXT();
+ }
+
EEO_CASE(EEOP_CASE_TESTVAL)
{
/*
@@ -2477,6 +2485,20 @@ ExecEvalParamExtern(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
errmsg("no value found for parameter %d", paramId)));
}
+/*
+ * Set value of a param (currently always PARAM_EXEC) from
+ * state->res{value,null}.
+ */
+void
+ExecEvalParamSet(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
+{
+ ParamExecData *prm;
+
+ prm = &(econtext->ecxt_param_exec_vals[op->d.param.paramid]);
+ prm->value = state->resvalue;
+ prm->isnull = state->resnull;
+}
+
/*
* Raise error if a CURRENT OF expression is evaluated.
*
diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c
index 4d288bc8d41..f62bb28140f 100644
--- a/src/backend/executor/execProcnode.c
+++ b/src/backend/executor/execProcnode.c
@@ -393,6 +393,10 @@ ExecInitNode(Plan *node, EState *estate, int eflags)
/*
* Initialize any initPlans present in this node. The planner put them in
* a separate list for us.
+ *
+ * The defining characteristic of initplans is that they don't have
+ * arguments, so we don't need to evaluate them (in contrast to
+ * ExecInitSubPlanExpr()).
*/
subps = NIL;
foreach(l, node->initPlan)
@@ -401,6 +405,7 @@ ExecInitNode(Plan *node, EState *estate, int eflags)
SubPlanState *sstate;
Assert(IsA(subplan, SubPlan));
+ Assert(subplan->args == NIL);
sstate = ExecInitSubPlan(subplan, result);
subps = lappend(subps, sstate);
}
diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c
index c136f75ac24..3458ac007cd 100644
--- a/src/backend/executor/nodeSubplan.c
+++ b/src/backend/executor/nodeSubplan.c
@@ -107,7 +107,7 @@ ExecHashSubPlan(SubPlanState *node,
TupleTableSlot *slot;
/* Shouldn't have any direct correlation Vars */
- if (subplan->parParam != NIL || node->args != NIL)
+ if (subplan->parParam != NIL || subplan->args != NIL)
elog(ERROR, "hashed subplan with direct correlation not supported");
/*
@@ -231,7 +231,6 @@ ExecScanSubPlan(SubPlanState *node,
TupleTableSlot *slot;
Datum result;
bool found = false; /* true if got at least one subplan tuple */
- ListCell *pvar;
ListCell *l;
ArrayBuildStateAny *astate = NULL;
@@ -248,26 +247,20 @@ ExecScanSubPlan(SubPlanState *node,
oldcontext = MemoryContextSwitchTo(econtext->ecxt_per_query_memory);
/*
- * Set Params of this plan from parent plan correlation values. (Any
- * calculation we have to do is done in the parent econtext, since the
- * Param values don't need to have per-query lifetime.)
+ * We rely on the caller to evaluate plan correlation values, if
+ * necessary. However we still need to record the fact that the values
+ * (might have) changed, otherwise the ExecReScan() below won't know that
+ * nodes need to be rescanned.
*/
- Assert(list_length(subplan->parParam) == list_length(node->args));
-
- forboth(l, subplan->parParam, pvar, node->args)
+ Assert(list_length(subplan->parParam) == list_length(subplan->args));
+ foreach(l, subplan->parParam)
{
int paramid = lfirst_int(l);
- ParamExecData *prm = &(econtext->ecxt_param_exec_vals[paramid]);
- prm->value = ExecEvalExprSwitchContext((ExprState *) lfirst(pvar),
- econtext,
- &(prm->isnull));
planstate->chgParam = bms_add_member(planstate->chgParam, paramid);
}
- /*
- * Now that we've set up its parameters, we can reset the subplan.
- */
+ /* with that done, we can reset the subplan */
ExecReScan(planstate);
/*
@@ -817,6 +810,10 @@ slotNoNulls(TupleTableSlot *slot)
* as well as regular SubPlans. Note that we don't link the SubPlan into
* the parent's subPlan list, because that shouldn't happen for InitPlans.
* Instead, ExecInitExpr() does that one part.
+ *
+ * We also rely on ExecInitExpr(), more precisely ExecInitSubPlanExpr(), to
+ * evaluate input parameters, as that allows them to be evaluated as part of
+ * the expression referencing the SubPlan.
* ----------------------------------------------------------------
*/
SubPlanState *
@@ -844,7 +841,6 @@ ExecInitSubPlan(SubPlan *subplan, PlanState *parent)
/* Initialize subexpressions */
sstate->testexpr = ExecInitExpr((Expr *) subplan->testexpr, parent);
- sstate->args = ExecInitExprList(subplan->args, parent);
/*
* initialize my state
@@ -1107,7 +1103,7 @@ ExecSetParamPlan(SubPlanState *node, ExprContext *econtext)
elog(ERROR, "ANY/ALL subselect unsupported as initplan");
if (subLinkType == CTE_SUBLINK)
elog(ERROR, "CTE subplans should not be executed via ExecSetParamPlan");
- if (subplan->parParam || node->args)
+ if (subplan->parParam || subplan->args)
elog(ERROR, "correlated subplans should not be executed via ExecSetParamPlan");
/*
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index 1c722c79552..812f8758743 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1094,6 +1094,12 @@ llvm_compile_expr(ExprState *state)
break;
}
+ case EEOP_PARAM_SET:
+ build_EvalXFunc(b, mod, "ExecEvalParamSet",
+ v_state, op, v_econtext);
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
case EEOP_SBSREF_SUBSCRIPTS:
{
int jumpdone = op->d.sbsref_subscript.jumpdone;
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 876fb640294..0b1df3afe00 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -123,6 +123,7 @@ void *referenced_functions[] =
ExecEvalNextValueExpr,
ExecEvalParamExec,
ExecEvalParamExtern,
+ ExecEvalParamSet,
ExecEvalRow,
ExecEvalRowNotNull,
ExecEvalRowNull,
--
2.37.1.188.g71a8fab31b
--2fo66m5ddj2bowyf--
^ permalink raw reply [nested|flat] 7+ messages in thread
* [PATCH v1] WIP: Evaluate arguments of correlated SubPlans in the referencing ExprState
@ 2023-02-25 21:39 Andres Freund <[email protected]>
0 siblings, 0 replies; 7+ messages in thread
From: Andres Freund @ 2023-02-25 21:39 UTC (permalink / raw)
---
src/include/nodes/execnodes.h | 1 -
src/backend/executor/execExpr.c | 81 +++++++++++++++++------------
src/backend/executor/execProcnode.c | 5 ++
src/backend/executor/nodeSubplan.c | 30 +++++------
4 files changed, 66 insertions(+), 51 deletions(-)
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 20f4c8b35f3..437cf8b5a02 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -947,7 +947,6 @@ typedef struct SubPlanState
struct PlanState *planstate; /* subselect plan's state tree */
struct PlanState *parent; /* parent plan node's state tree */
ExprState *testexpr; /* state of combining expression */
- List *args; /* states of argument expression(s) */
HeapTuple curTuple; /* copy of most recent tuple from subplan */
Datum curArray; /* most recent array from ARRAY() subplan */
/* these are used when hashing the subselect's output: */
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index c61f23c6c18..7a9d5729b4b 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -87,6 +87,9 @@ static void ExecBuildAggTransCall(ExprState *state, AggState *aggstate,
FunctionCallInfo fcinfo, AggStatePerTrans pertrans,
int transno, int setno, int setoff, bool ishash,
bool nullcheck);
+static void ExecInitSubPlanExpr(SubPlan *subplan,
+ ExprState *state,
+ Datum *resv, bool *resnull);
/*
@@ -1388,7 +1391,6 @@ ExecInitExprRec(Expr *node, ExprState *state,
case T_SubPlan:
{
SubPlan *subplan = (SubPlan *) node;
- SubPlanState *sstate;
/*
* Real execution of a MULTIEXPR SubPlan has already been
@@ -1405,19 +1407,7 @@ ExecInitExprRec(Expr *node, ExprState *state,
break;
}
- if (!state->parent)
- elog(ERROR, "SubPlan found with no parent plan");
-
- sstate = ExecInitSubPlan(subplan, state->parent);
-
- /* add SubPlanState nodes to state->parent->subPlan */
- state->parent->subPlan = lappend(state->parent->subPlan,
- sstate);
-
- scratch.opcode = EEOP_SUBPLAN;
- scratch.d.subplan.sstate = sstate;
-
- ExprEvalPushStep(state, &scratch);
+ ExecInitSubPlanExpr(subplan, state, resv, resnull);
break;
}
@@ -2618,29 +2608,12 @@ ExecPushExprSetupSteps(ExprState *state, ExprSetupInfo *info)
foreach(lc, info->multiexpr_subplans)
{
SubPlan *subplan = (SubPlan *) lfirst(lc);
- SubPlanState *sstate;
Assert(subplan->subLinkType == MULTIEXPR_SUBLINK);
- /* This should match what ExecInitExprRec does for other SubPlans: */
-
- if (!state->parent)
- elog(ERROR, "SubPlan found with no parent plan");
-
- sstate = ExecInitSubPlan(subplan, state->parent);
-
- /* add SubPlanState nodes to state->parent->subPlan */
- state->parent->subPlan = lappend(state->parent->subPlan,
- sstate);
-
- scratch.opcode = EEOP_SUBPLAN;
- scratch.d.subplan.sstate = sstate;
-
/* The result can be ignored, but we better put it somewhere */
- scratch.resvalue = &state->resvalue;
- scratch.resnull = &state->resnull;
-
- ExprEvalPushStep(state, &scratch);
+ ExecInitSubPlanExpr(subplan, state,
+ &state->resvalue, &state->resnull);
}
}
@@ -4040,3 +4013,45 @@ ExecBuildParamSetEqual(TupleDesc desc,
return state;
}
+
+static void
+ExecInitSubPlanExpr(SubPlan *subplan,
+ ExprState *state,
+ Datum *resv, bool *resnull)
+{
+ ExprEvalStep scratch = {0};
+ SubPlanState *sstate;
+ ListCell *pvar;
+ ListCell *l;
+ EState *estate = state->parent->state;
+
+ if (!state->parent)
+ elog(ERROR, "SubPlan found with no parent plan");
+
+ /*
+ * Generate steps to evaluate input arguments for the subplan.
+ *
+ * Any calculation we have to do can be done in the parent econtext, since
+ * the Param values don't need to have per-query lifetime.
+ */
+ forboth(l, subplan->parParam, pvar, subplan->args)
+ {
+ int paramid = lfirst_int(l);
+ ParamExecData *prm = &estate->es_param_exec_vals[paramid];
+
+ ExecInitExprRec(lfirst(pvar), state, &prm->value, &prm->isnull);
+ }
+
+ sstate = ExecInitSubPlan(subplan, state->parent);
+
+ /* add SubPlanState nodes to state->parent->subPlan */
+ state->parent->subPlan = lappend(state->parent->subPlan,
+ sstate);
+
+ scratch.opcode = EEOP_SUBPLAN;
+ scratch.resvalue = resv;
+ scratch.resnull = resnull;
+ scratch.d.subplan.sstate = sstate;
+
+ ExprEvalPushStep(state, &scratch);
+}
diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c
index 4d288bc8d41..f62bb28140f 100644
--- a/src/backend/executor/execProcnode.c
+++ b/src/backend/executor/execProcnode.c
@@ -393,6 +393,10 @@ ExecInitNode(Plan *node, EState *estate, int eflags)
/*
* Initialize any initPlans present in this node. The planner put them in
* a separate list for us.
+ *
+ * The defining characteristic of initplans is that they don't have
+ * arguments, so we don't need to evaluate them (in contrast to
+ * ExecInitSubPlanExpr()).
*/
subps = NIL;
foreach(l, node->initPlan)
@@ -401,6 +405,7 @@ ExecInitNode(Plan *node, EState *estate, int eflags)
SubPlanState *sstate;
Assert(IsA(subplan, SubPlan));
+ Assert(subplan->args == NIL);
sstate = ExecInitSubPlan(subplan, result);
subps = lappend(subps, sstate);
}
diff --git a/src/backend/executor/nodeSubplan.c b/src/backend/executor/nodeSubplan.c
index c136f75ac24..3458ac007cd 100644
--- a/src/backend/executor/nodeSubplan.c
+++ b/src/backend/executor/nodeSubplan.c
@@ -107,7 +107,7 @@ ExecHashSubPlan(SubPlanState *node,
TupleTableSlot *slot;
/* Shouldn't have any direct correlation Vars */
- if (subplan->parParam != NIL || node->args != NIL)
+ if (subplan->parParam != NIL || subplan->args != NIL)
elog(ERROR, "hashed subplan with direct correlation not supported");
/*
@@ -231,7 +231,6 @@ ExecScanSubPlan(SubPlanState *node,
TupleTableSlot *slot;
Datum result;
bool found = false; /* true if got at least one subplan tuple */
- ListCell *pvar;
ListCell *l;
ArrayBuildStateAny *astate = NULL;
@@ -248,26 +247,20 @@ ExecScanSubPlan(SubPlanState *node,
oldcontext = MemoryContextSwitchTo(econtext->ecxt_per_query_memory);
/*
- * Set Params of this plan from parent plan correlation values. (Any
- * calculation we have to do is done in the parent econtext, since the
- * Param values don't need to have per-query lifetime.)
+ * We rely on the caller to evaluate plan correlation values, if
+ * necessary. However we still need to record the fact that the values
+ * (might have) changed, otherwise the ExecReScan() below won't know that
+ * nodes need to be rescanned.
*/
- Assert(list_length(subplan->parParam) == list_length(node->args));
-
- forboth(l, subplan->parParam, pvar, node->args)
+ Assert(list_length(subplan->parParam) == list_length(subplan->args));
+ foreach(l, subplan->parParam)
{
int paramid = lfirst_int(l);
- ParamExecData *prm = &(econtext->ecxt_param_exec_vals[paramid]);
- prm->value = ExecEvalExprSwitchContext((ExprState *) lfirst(pvar),
- econtext,
- &(prm->isnull));
planstate->chgParam = bms_add_member(planstate->chgParam, paramid);
}
- /*
- * Now that we've set up its parameters, we can reset the subplan.
- */
+ /* with that done, we can reset the subplan */
ExecReScan(planstate);
/*
@@ -817,6 +810,10 @@ slotNoNulls(TupleTableSlot *slot)
* as well as regular SubPlans. Note that we don't link the SubPlan into
* the parent's subPlan list, because that shouldn't happen for InitPlans.
* Instead, ExecInitExpr() does that one part.
+ *
+ * We also rely on ExecInitExpr(), more precisely ExecInitSubPlanExpr(), to
+ * evaluate input parameters, as that allows them to be evaluated as part of
+ * the expression referencing the SubPlan.
* ----------------------------------------------------------------
*/
SubPlanState *
@@ -844,7 +841,6 @@ ExecInitSubPlan(SubPlan *subplan, PlanState *parent)
/* Initialize subexpressions */
sstate->testexpr = ExecInitExpr((Expr *) subplan->testexpr, parent);
- sstate->args = ExecInitExprList(subplan->args, parent);
/*
* initialize my state
@@ -1107,7 +1103,7 @@ ExecSetParamPlan(SubPlanState *node, ExprContext *econtext)
elog(ERROR, "ANY/ALL subselect unsupported as initplan");
if (subLinkType == CTE_SUBLINK)
elog(ERROR, "CTE subplans should not be executed via ExecSetParamPlan");
- if (subplan->parParam || node->args)
+ if (subplan->parParam || subplan->args)
elog(ERROR, "correlated subplans should not be executed via ExecSetParamPlan");
/*
--
2.38.0
--6ir4gcdcv6d4j3qy--
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Align memory context level numbering in pg_log_backend_memory_contexts()
@ 2025-04-15 21:18 Daniel Gustafsson <[email protected]>
2025-04-16 14:19 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() torikoshia <[email protected]>
0 siblings, 1 reply; 7+ messages in thread
From: Daniel Gustafsson @ 2025-04-15 21:18 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: torikoshia <[email protected]>; pgsql-hackers
> On 15 Apr 2025, at 23:03, David Rowley <[email protected]> wrote:
> My vote is to make the levels 1-based in all locations where we output
> the context information.
I agree with this, pg_get_process_memory_contexts() also use 1-based levels
fwiw.
--
Daniel Gustafsson
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Align memory context level numbering in pg_log_backend_memory_contexts()
2025-04-15 21:18 Re: Align memory context level numbering in pg_log_backend_memory_contexts() Daniel Gustafsson <[email protected]>
@ 2025-04-16 14:19 ` torikoshia <[email protected]>
2025-04-17 14:27 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() torikoshia <[email protected]>
2025-04-18 09:45 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() Fujii Masao <[email protected]>
0 siblings, 2 replies; 7+ messages in thread
From: torikoshia @ 2025-04-16 14:19 UTC (permalink / raw)
To: Daniel Gustafsson <[email protected]>; [email protected]; +Cc: pgsql-hackers
On 2025-04-16 06:18, Daniel Gustafsson wrote:
>> On 15 Apr 2025, at 23:03, David Rowley <[email protected]> wrote:
>
>> My vote is to make the levels 1-based in all locations where we output
>> the context information.
>
> I agree with this, pg_get_process_memory_contexts() also use 1-based
> levels
> fwiw.
+1.
I believe there's no particular issue with starting the level from 1 in
pg_log_backend_memory_contexts().
Regarding the implementation:
In the initial patch attached, I naïvely incremented the level just
before emitting the log line.
However, it might be cleaner to simply initialize the level variable to
1 from the start. This could help avoid unnecessary confusion when
debugging that part of the code.
Similarly, I noticed that in pg_get_process_memory_contexts(), the level
is initialized to 0 in ProcessGetMemoryContextInterrupt(void):
int level = 0;
..
MemoryContextStatsInternal(c, level, 100, 100, &grand_totals, ..
If we want to be consistent, perhaps it would make sense to start from 1
there as well.
BTW level variable has existed since before pg_backend_memory_contexts
was introduced — it was originally used for functions that help inspect
memory contexts via the debugger. Because of that, I think changing this
would affect not only these functions codes but some older ones.
--
Atsushi Torikoshi
Seconded from NTT DATA GROUP CORPORATION to SRA OSS K.K.
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Align memory context level numbering in pg_log_backend_memory_contexts()
2025-04-15 21:18 Re: Align memory context level numbering in pg_log_backend_memory_contexts() Daniel Gustafsson <[email protected]>
2025-04-16 14:19 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() torikoshia <[email protected]>
@ 2025-04-17 14:27 ` torikoshia <[email protected]>
1 sibling, 0 replies; 7+ messages in thread
From: torikoshia @ 2025-04-17 14:27 UTC (permalink / raw)
To: Rahila Syed <[email protected]>; [email protected]; +Cc: pgsql-hackers
Thanks for your review, Melih and Rahila.
On 2025-04-17 21:25, Rahila Syed wrote:
> Hi,
>
>> The attached patch is how I think we should do it.
>
> Thank you for the patch.
> I tested this patch and it works fine. I agree with the changes made
> in it.
>
> Regarding v2 patch,
> - int level = 0;
>
> Retaining the level variable will enhance the code readability, IMO.
As for the level variable, this change comes from the v1 patch, and I
don't have a strong opinion about it.
However, if we decide to keep the level variable here, it might be more
consistent to also define it in MemoryContextStatsDetail().
--
Regards,
--
Atsushi Torikoshi
Seconded from NTT DATA GROUP CORPORATION to SRA OSS K.K.
^ permalink raw reply [nested|flat] 7+ messages in thread
* Re: Align memory context level numbering in pg_log_backend_memory_contexts()
2025-04-15 21:18 Re: Align memory context level numbering in pg_log_backend_memory_contexts() Daniel Gustafsson <[email protected]>
2025-04-16 14:19 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() torikoshia <[email protected]>
@ 2025-04-18 09:45 ` Fujii Masao <[email protected]>
1 sibling, 0 replies; 7+ messages in thread
From: Fujii Masao @ 2025-04-18 09:45 UTC (permalink / raw)
To: David Rowley <[email protected]>; +Cc: Rahila Syed <[email protected]>; pgsql-hackers
On 2025/04/18 18:23, David Rowley wrote:
> On Fri, 18 Apr 2025 at 20:54, Fujii Masao <[email protected]> wrote:
>> Shouldn't the example output of pg_log_backend_memory_contexts() in
>> the documentation also be updated to use 1-based numbering for consistency?
>> Patch attached.
>
> Yeah. I failed to notice we had an example of the output.
>
> Want to take care of it?
Yeah, I will if you're okay with that!
Regards,
--
Fujii Masao
Advanced Computing Technology Center
Research and Development Headquarters
NTT DATA CORPORATION
^ permalink raw reply [nested|flat] 7+ messages in thread
end of thread, other threads:[~2025-04-18 09:45 UTC | newest]
Thread overview: 7+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2023-02-25 21:39 [PATCH v2] WIP: Evaluate arguments of correlated SubPlans in the referencing ExprState Andres Freund <[email protected]>
2023-02-25 21:39 [PATCH v1] WIP: Evaluate arguments of correlated SubPlans in the referencing ExprState Andres Freund <[email protected]>
2025-04-15 21:18 Re: Align memory context level numbering in pg_log_backend_memory_contexts() Daniel Gustafsson <[email protected]>
2025-04-16 14:19 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() torikoshia <[email protected]>
2025-04-17 14:27 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() torikoshia <[email protected]>
2025-04-18 09:45 ` Re: Align memory context level numbering in pg_log_backend_memory_contexts() Fujii Masao <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox