public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v7 2/2] Add a --outdated option to reindexdb
94+ messages / 6 participants
[nested] [flat]
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v7 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency using a new
pg_index_has_outdated_dependency(regclass) SQL function.
Catversion (should be) bumped.
Author: Julien Rouhaud <[email protected]>
Reviewed-by: Michael Paquier, Mark Dilger
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
doc/src/sgml/func.sgml | 27 ++++--
src/backend/catalog/index.c | 22 +++++
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
src/include/catalog/pg_proc.dat | 4 +
5 files changed, 198 insertions(+), 32 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9492a3c6b9..0eda6678ac 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26448,12 +26448,13 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
<para>
<xref linkend="functions-admin-index-table"/> shows the functions
- available for index maintenance tasks. (Note that these maintenance
- tasks are normally done automatically by autovacuum; use of these
- functions is only required in special cases.)
- These functions cannot be executed during recovery.
- Use of these functions is restricted to superusers and the owner
- of the given index.
+ available for index maintenance tasks. (Note that the maintenance
+ tasks performing actions on indexes are normally done automatically by
+ autovacuum; use of these functions is only required in special cases.)
+ The functions performing actions on indexes cannot be executed during
+ recovery.
+ Use of the functions performing actions on indexes is restricted to
+ superusers and the owner of the given index.
</para>
<table id="functions-admin-index-table">
@@ -26538,6 +26539,20 @@ SELECT pg_size_pretty(sum(pg_relation_size(relid))) AS total_size
option.
</para></entry>
</row>
+
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm>
+ <primary>pg_index_has_outdated_dependency</primary>
+ </indexterm>
+ <function>pg_index_has_outdated_dependency</function> ( <parameter>index</parameter> <type>regclass</type> )
+ <returnvalue>boolean</returnvalue>
+ </para>
+ <para>
+ Check if the specified index has any outdated dependency. For now only
+ dependency on collations are supported.
+ </para></entry>
+ </row>
</tbody>
</tgroup>
</table>
diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c
index 47e6a54149..6848e61df3 100644
--- a/src/backend/catalog/index.c
+++ b/src/backend/catalog/index.c
@@ -1390,6 +1390,28 @@ do_check_index_has_outdated_collation(const ObjectAddress *otherObject,
return false;
}
+/*
+ * SQL wrapper around index_has_outdated_dependency.
+ */
+Datum
+pg_index_has_outdated_dependency(PG_FUNCTION_ARGS)
+{
+ Oid indexOid = PG_GETARG_OID(0);
+ Relation rel;
+ bool res;
+
+ rel = try_index_open(indexOid, AccessShareLock);
+
+ if (rel == NULL)
+ PG_RETURN_NULL();
+
+ res = index_has_outdated_dependency(indexOid);
+
+ relation_close(rel, AccessShareLock);
+
+ PG_RETURN_BOOL(res);
+}
+
/*
* Check whether the given index has a dependency with an outdated
* collation version.
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..d252b1454f 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY pg_catalog.sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated dependencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..91fd602041 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/index "t2_id_idx" has no outdated dependency/],
+ 'verbose reindexdb for outdated dependencies database wide reports all ignored indexes');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 93393fcfd4..911c12ee2c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -949,6 +949,10 @@
proname => 'pg_indexam_has_property', provolatile => 's',
prorettype => 'bool', proargtypes => 'oid text',
prosrc => 'pg_indexam_has_property' },
+{ oid => '8102', descr => 'test property of an index',
+ proname => 'pg_index_has_outdated_dependency', provolatile => 's',
+ prorettype => 'bool', proargtypes => 'regclass',
+ prosrc => 'pg_index_has_outdated_dependency' },
{ oid => '637', descr => 'test property of an index',
proname => 'pg_index_has_property', provolatile => 's', prorettype => 'bool',
proargtypes => 'regclass text', prosrc => 'pg_index_has_property' },
--
2.30.1
--rpt4sswdwkhtp4fw--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v5 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index cf28176243..369d164e65 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -484,7 +521,8 @@ reindex_one_database(const ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -509,7 +547,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -536,6 +574,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -641,7 +685,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -660,16 +704,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -687,7 +756,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -705,8 +785,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -754,7 +846,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -778,7 +870,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -797,6 +889,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--oao5se3im4osqowp--
^ permalink raw reply [nested|flat] 94+ messages in thread
* [PATCH v6 2/2] Add a --outdated option to reindexdb
@ 2021-02-24 17:33 Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Julien Rouhaud @ 2021-02-24 17:33 UTC (permalink / raw)
This uses the new OUTDATED option for REINDEX. If user asks for multiple job,
the list of tables to process will be sorted by the total size of underlying
indexes that have outdated dependency.
Author: Julien Rouhaud <[email protected]>
Reviewed-by:
Discussion: https://postgr.es/m/20201203093143.GA64934%40nol
---
src/bin/scripts/reindexdb.c | 143 ++++++++++++++++++++++++-----
src/bin/scripts/t/090_reindexdb.pl | 34 ++++++-
2 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/src/bin/scripts/reindexdb.c b/src/bin/scripts/reindexdb.c
index fc0681538a..addea5c1ed 100644
--- a/src/bin/scripts/reindexdb.c
+++ b/src/bin/scripts/reindexdb.c
@@ -35,20 +35,23 @@ typedef enum ReindexType
static SimpleStringList *get_parallel_object_list(PGconn *conn,
ReindexType type,
SimpleStringList *user_list,
+ bool outdated,
bool echo);
static void reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname,
bool echo, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo,
bool quiet, bool verbose, bool concurrently,
- int concurrentCons, const char *tablespace);
+ int concurrentCons, const char *tablespace,
+ bool outdated);
static void run_reindex_command(PGconn *conn, ReindexType type,
const char *name, bool echo, bool verbose,
bool concurrently, bool async,
- const char *tablespace);
+ const char *tablespace, bool outdated);
static void help(const char *progname);
@@ -74,6 +77,7 @@ main(int argc, char *argv[])
{"concurrently", no_argument, NULL, 1},
{"maintenance-db", required_argument, NULL, 2},
{"tablespace", required_argument, NULL, 3},
+ {"outdated", no_argument, NULL, 4},
{NULL, 0, NULL, 0}
};
@@ -95,6 +99,7 @@ main(int argc, char *argv[])
bool quiet = false;
bool verbose = false;
bool concurrently = false;
+ bool outdated = false;
SimpleStringList indexes = {NULL, NULL};
SimpleStringList tables = {NULL, NULL};
SimpleStringList schemas = {NULL, NULL};
@@ -170,6 +175,9 @@ main(int argc, char *argv[])
case 3:
tablespace = pg_strdup(optarg);
break;
+ case 4:
+ outdated = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -234,7 +242,8 @@ main(int argc, char *argv[])
cparams.dbname = maintenance_db;
reindex_all_databases(&cparams, progname, echo, quiet, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
else if (syscatalog)
{
@@ -253,12 +262,17 @@ main(int argc, char *argv[])
pg_log_error("cannot reindex specific index(es) and system catalogs at the same time");
exit(1);
}
-
if (concurrentCons > 1)
{
pg_log_error("cannot use multiple jobs to reindex system catalogs");
exit(1);
}
+ if (outdated)
+ {
+ pg_log_error("cannot filter indexes having outdated dependencies "
+ "and reindex system catalogs at the same time");
+ exit(1);
+ }
if (dbname == NULL)
{
@@ -274,7 +288,7 @@ main(int argc, char *argv[])
reindex_one_database(&cparams, REINDEX_SYSTEM, NULL,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace, outdated);
}
else
{
@@ -304,17 +318,20 @@ main(int argc, char *argv[])
if (schemas.head != NULL)
reindex_one_database(&cparams, REINDEX_SCHEMA, &schemas,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
if (indexes.head != NULL)
reindex_one_database(&cparams, REINDEX_INDEX, &indexes,
progname, echo, verbose,
- concurrently, 1, tablespace);
+ concurrently, 1, tablespace,
+ outdated);
if (tables.head != NULL)
reindex_one_database(&cparams, REINDEX_TABLE, &tables,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
/*
* reindex database only if neither index nor table nor schema is
@@ -323,7 +340,8 @@ main(int argc, char *argv[])
if (indexes.head == NULL && tables.head == NULL && schemas.head == NULL)
reindex_one_database(&cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose,
- concurrently, concurrentCons, tablespace);
+ concurrently, concurrentCons, tablespace,
+ outdated);
}
exit(0);
@@ -334,7 +352,7 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
SimpleStringList *user_list,
const char *progname, bool echo,
bool verbose, bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
SimpleStringListCell *cell;
@@ -363,6 +381,14 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
exit(1);
}
+ if (outdated && PQserverVersion(conn) < 140000)
+ {
+ PQfinish(conn);
+ pg_log_error("cannot use the \"%s\" option on server versions older than PostgreSQL %s",
+ "outdated", "14");
+ exit(1);
+ }
+
if (!parallel)
{
switch (process_type)
@@ -399,14 +425,24 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
*/
if (concurrently)
pg_log_warning("cannot reindex system catalogs concurrently, skipping all");
+ else if (outdated)
+ {
+ /*
+ * The only supported kind of object that can be outdated
+ * is collation. No system catalog has any index that can
+ * depend on an outdated collation, so skip system
+ * catalogs.
+ */
+ }
else
run_reindex_command(conn, REINDEX_SYSTEM, PQdb(conn), echo,
verbose, concurrently, false,
- tablespace);
+ tablespace, outdated);
/* Build a list of relations from the database */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -419,7 +455,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
/* Build a list of relations from all the schemas */
process_list = get_parallel_object_list(conn, process_type,
- user_list, echo);
+ user_list, outdated,
+ echo);
process_type = REINDEX_TABLE;
/* Bail out if nothing to process */
@@ -485,7 +522,8 @@ reindex_one_database(ConnParams *cparams, ReindexType type,
ParallelSlotSetHandler(free_slot, TableCommandResultHandler, NULL);
run_reindex_command(free_slot->connection, process_type, objname,
- echo, verbose, concurrently, true, tablespace);
+ echo, verbose, concurrently, true, tablespace,
+ outdated);
cell = cell->next;
} while (cell != NULL);
@@ -510,7 +548,7 @@ finish:
static void
run_reindex_command(PGconn *conn, ReindexType type, const char *name,
bool echo, bool verbose, bool concurrently, bool async,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
const char *paren = "(";
const char *comma = ", ";
@@ -537,6 +575,12 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
sep = comma;
}
+ if (outdated)
+ {
+ appendPQExpBuffer(&sql, "%sOUTDATED", sep);
+ sep = comma;
+ }
+
if (sep != paren)
appendPQExpBufferStr(&sql, ") ");
@@ -642,7 +686,7 @@ run_reindex_command(PGconn *conn, ReindexType type, const char *name,
*/
static SimpleStringList *
get_parallel_object_list(PGconn *conn, ReindexType type,
- SimpleStringList *user_list, bool echo)
+ SimpleStringList *user_list, bool outdated, bool echo)
{
PQExpBufferData catalog_query;
PQExpBufferData buf;
@@ -661,16 +705,41 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
{
case REINDEX_DATABASE:
Assert(user_list == NULL);
+
appendPQExpBufferStr(&catalog_query,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE ns.nspname != 'pg_catalog'\n"
" AND c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
- CppAsString2(RELKIND_MATVIEW) ")\n"
- " ORDER BY c.relpages DESC;");
+ CppAsString2(RELKIND_MATVIEW) ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
+
break;
case REINDEX_SCHEMA:
@@ -688,7 +757,18 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
"SELECT c.relname, ns.nspname\n"
" FROM pg_catalog.pg_class c\n"
" JOIN pg_catalog.pg_namespace ns"
- " ON c.relnamespace = ns.oid\n"
+ " ON c.relnamespace = ns.oid\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " JOIN pg_catalog.pg_index i"
+ " ON c.oid = i.indrelid\n"
+ " JOIN pg_catalog.pg_class ci"
+ " ON i.indexrelid = ci.oid\n");
+ }
+
+ appendPQExpBufferStr(&catalog_query,
" WHERE c.relkind IN ("
CppAsString2(RELKIND_RELATION) ", "
CppAsString2(RELKIND_MATVIEW) ")\n"
@@ -706,8 +786,20 @@ get_parallel_object_list(PGconn *conn, ReindexType type,
appendStringLiteralConn(&catalog_query, nspname, conn);
}
- appendPQExpBufferStr(&catalog_query, ")\n"
- " ORDER BY c.relpages DESC;");
+ appendPQExpBufferStr(&catalog_query, ")\n");
+
+ if (outdated)
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " GROUP BY c.relname, ns.nspname\n"
+ " ORDER BY sum(ci.relpages)"
+ " FILTER (WHERE pg_catalog.pg_index_has_outdated_dependency(ci.oid)) DESC;");
+ }
+ else
+ {
+ appendPQExpBufferStr(&catalog_query,
+ " ORDER BY c.relpages DESC;");
+ }
}
break;
@@ -755,7 +847,7 @@ static void
reindex_all_databases(ConnParams *cparams,
const char *progname, bool echo, bool quiet, bool verbose,
bool concurrently, int concurrentCons,
- const char *tablespace)
+ const char *tablespace, bool outdated)
{
PGconn *conn;
PGresult *result;
@@ -779,7 +871,7 @@ reindex_all_databases(ConnParams *cparams,
reindex_one_database(cparams, REINDEX_DATABASE, NULL,
progname, echo, verbose, concurrently,
- concurrentCons, tablespace);
+ concurrentCons, tablespace, outdated);
}
PQclear(result);
@@ -798,6 +890,7 @@ help(const char *progname)
printf(_(" -e, --echo show the commands being sent to the server\n"));
printf(_(" -i, --index=INDEX recreate specific index(es) only\n"));
printf(_(" -j, --jobs=NUM use this many concurrent connections to reindex\n"));
+ printf(_(" --outdated only process indexes having outdated depencies\n"));
printf(_(" -q, --quiet don't write any messages\n"));
printf(_(" -s, --system reindex system catalogs\n"));
printf(_(" -S, --schema=SCHEMA reindex specific schema(s) only\n"));
diff --git a/src/bin/scripts/t/090_reindexdb.pl b/src/bin/scripts/t/090_reindexdb.pl
index 159b637230..b60cac6081 100644
--- a/src/bin/scripts/t/090_reindexdb.pl
+++ b/src/bin/scripts/t/090_reindexdb.pl
@@ -3,7 +3,7 @@ use warnings;
use PostgresNode;
use TestLib;
-use Test::More tests => 58;
+use Test::More tests => 70;
program_help_ok('reindexdb');
program_version_ok('reindexdb');
@@ -174,6 +174,9 @@ $node->command_fails(
$node->command_fails(
[ 'reindexdb', '-j', '2', '-i', 'i1', 'postgres' ],
'parallel reindexdb cannot process indexes');
+$node->command_fails(
+ [ 'reindexdb', '-s', '--outdated' ],
+ 'cannot reindex system catalog and filter indexes having outdated dependencies');
$node->issues_sql_like(
[ 'reindexdb', '-j', '2', 'postgres' ],
qr/statement:\ REINDEX SYSTEM postgres;
@@ -196,3 +199,32 @@ $node->command_checks_all(
qr/^reindexdb: warning: cannot reindex system catalogs concurrently, skipping all/s
],
'parallel reindexdb for system with --concurrently skips catalogs');
+
+# Temporarily downgrade client-min-message to get the no-op report
+$ENV{PGOPTIONS} = '--client-min-messages=NOTICE';
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-t', 's1.t1', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'verbose reindexdb for outdated dependencies on a specific table reports no-op tables');
+
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/^$/],
+ 'verbose reindexdb for outdated dependencies database wide silently ignore all tables');
+$node->command_checks_all(
+ [ 'reindexdb', '--outdated', '-v', '-j', '2', '-d', 'postgres' ],
+ 0,
+ [qr/^$/],
+ [qr/table "t1" has no indexes to reindex/],
+ 'parallel verbose reindexdb for outdated dependencies database wide reports no-op tables');
+
+# Switch back to WARNING client-min-message
+$ENV{PGOPTIONS} = '--client-min-messages=WARNING';
+$node->issues_sql_like(
+ [ 'reindexdb', '--outdated', '-t', 's1.t1', 'postgres' ],
+ qr/.*statement: REINDEX \(OUTDATED\) TABLE s1\.t1;/,
+ 'reindexdb for outdated dependencies specify the OUTDATED keyword');
--
2.30.1
--mzh3msf6semlblkg--
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
@ 2023-09-29 04:57 Amit Langote <[email protected]>
2023-10-02 04:24 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
0 siblings, 2 replies; 94+ messages in thread
From: Amit Langote @ 2023-09-29 04:57 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
On Thu, Sep 28, 2023 at 8:04 PM Alvaro Herrera <[email protected]> wrote:
> On 2023-Sep-27, Amit Langote wrote:
> > Maybe the following is better:
> >
> > + /*
> > + * For expression nodes that support soft errors. Should be set to NULL
> > + * before calling ExecInitExprRec() if the caller wants errors thrown.
> > + */
> >
> > ...as in the attached.
>
> That's good.
>
> > Alvaro, do you think your concern regarding escontext not being in the
> > right spot in the ExprState struct is addressed? It doesn't seem very
> > critical to me to place it in the struct's 1st cacheline, because
> > escontext is not accessed in performance critical paths such as during
> > expression evaluation, especially with the latest version. (It would
> > get accessed during evaluation with previous versions.)
> >
> > If so, I'd like to move ahead with committing it.
>
> Yeah, looks OK to me in v21.
Thanks. I will push the attached 0001 shortly.
Also, I've updated 0002's commit message to mention why it only
changes the functions local to jsonfuncs.c to add the Node *escontext
parameter, but not any external ones that may be invoked, such as,
makeMdArrayResult(). The assumption behind that is that jsonfuncs.c
functions validate any data that they pass to such external functions,
so no *suppressible* errors should occur.
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v22-0002-Add-soft-error-handling-to-populate_record_field.patch (23.2K, ../../CA+HiwqETtgH0ESDEEGDwTmazmtNeR7UsJS=BOoa=ghdYMo+Ufg@mail.gmail.com/2-v22-0002-Add-soft-error-handling-to-populate_record_field.patch)
download | inline diff:
From 397bee194c9204b6dfab4c486127b72ba5a0ffc4 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Fri, 29 Sep 2023 13:41:28 +0900
Subject: [PATCH v22 2/5] Add soft error handling to populate_record_field()
An uncoming patch would like the ability to call it from the
executor for some SQL/JSON expression nodes and ask to suppress any
errors that may occur.
This commit does two things mainly:
* It modifies the various interfaces internal to jsonfuncs.c to pass
the ErrorSaveContext around.
* Make necessary modifications to handle the cases where the
processing is aborted partway through various functions that take
an ErrorSaveContext when a soft error occurs.
Note that the above changes are only intended to suppress errors in
the functions in jsonfuncs.c, but not those in any external functions
that the functions in jsonfuncs.c in turn call, such as those from
arrayfuncs.c. It is assumed that the various populate_* functions
validate the data before passing those to external functions.
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/utils/adt/jsonfuncs.c | 280 ++++++++++++++++++++++--------
1 file changed, 210 insertions(+), 70 deletions(-)
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index a4bfa5e404..a8f2b186c9 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -265,6 +265,7 @@ typedef struct PopulateArrayContext
int *dims; /* dimensions */
int *sizes; /* current dimension counters */
int ndims; /* number of dimensions */
+ Node *escontext; /* For soft-error handling */
} PopulateArrayContext;
/* state for populate_array_json() */
@@ -389,7 +390,8 @@ static JsonParseErrorType elements_array_element_end(void *state, bool isnull);
static JsonParseErrorType elements_scalar(void *state, char *token, JsonTokenType tokentype);
/* turn a json object into a hash table */
-static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname);
+static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext);
/* semantic actions for populate_array_json */
static JsonParseErrorType populate_array_object_start(void *_state);
@@ -431,37 +433,42 @@ static Datum populate_record_worker(FunctionCallInfo fcinfo, const char *funcnam
/* helper functions for populate_record[set] */
static HeapTupleHeader populate_record(TupleDesc tupdesc, RecordIOData **record_p,
HeapTupleHeader defaultval, MemoryContext mcxt,
- JsObject *obj);
+ JsObject *obj, Node *escontext);
static void get_record_type_from_argument(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
static void get_record_type_from_query(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
-static void JsValueToJsObject(JsValue *jsv, JsObject *jso);
+static bool JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext);
static Datum populate_composite(CompositeIOData *io, Oid typid,
const char *colname, MemoryContext mcxt,
- HeapTupleHeader defaultval, JsValue *jsv, bool isnull);
-static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv);
+ HeapTupleHeader defaultval, JsValue *jsv, bool *isnull,
+ Node *escontext);
+static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext);
static void prepare_column_cache(ColumnIOData *column, Oid typid, int32 typmod,
MemoryContext mcxt, bool need_scalar);
static Datum populate_record_field(ColumnIOData *col, Oid typid, int32 typmod,
const char *colname, MemoryContext mcxt, Datum defaultval,
- JsValue *jsv, bool *isnull);
+ JsValue *jsv, bool *isnull, Node *escontext);
static RecordIOData *allocate_record_info(MemoryContext mcxt, int ncolumns);
static bool JsObjectGetField(JsObject *obj, char *field, JsValue *jsv);
static void populate_recordset_record(PopulateRecordsetState *state, JsObject *obj);
-static void populate_array_json(PopulateArrayContext *ctx, char *json, int len);
-static void populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
+static bool populate_array_json(PopulateArrayContext *ctx, char *json, int len);
+static bool populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
int ndim);
static void populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim);
-static void populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
-static void populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
-static void populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
+static bool populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
+static bool populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
+static bool populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
static Datum populate_array(ArrayIOData *aio, const char *colname,
- MemoryContext mcxt, JsValue *jsv);
+ MemoryContext mcxt, JsValue *jsv,
+ bool *isnull,
+ Node *escontext);
static Datum populate_domain(DomainIOData *io, Oid typid, const char *colname,
- MemoryContext mcxt, JsValue *jsv, bool isnull);
+ MemoryContext mcxt, JsValue *jsv, bool isnull,
+ Node *escontext);
/* functions supporting jsonb_delete, jsonb_set and jsonb_concat */
static JsonbValue *IteratorConcat(JsonbIterator **it1, JsonbIterator **it2,
@@ -2484,12 +2491,12 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
if (ndim <= 0)
{
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the value of key \"%s\".", ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array")));
}
@@ -2506,13 +2513,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
appendStringInfo(&indices, "[%d]", ctx->sizes[i]);
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s of key \"%s\".",
indices.data, ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s.",
@@ -2520,8 +2527,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
}
}
-/* set the number of dimensions of the populated array when it becomes known */
-static void
+/*
+ * Validate and set ndims for populating an array with some
+ * populate_array_*() function.
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
{
int i;
@@ -2529,7 +2541,12 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
Assert(ctx->ndims <= 0);
if (ndims <= 0)
+ {
populate_array_report_expected_array(ctx, ndims);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
ctx->ndims = ndims;
ctx->dims = palloc(sizeof(int) * ndims);
@@ -2537,10 +2554,16 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
for (i = 0; i < ndims; i++)
ctx->dims[i] = -1; /* dimensions are unknown yet */
+
+ return true;
}
-/* check the populated subarray dimension */
-static void
+/*
+ * Check the populated subarray dimension
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
{
int dim = ctx->sizes[ndim]; /* current dimension counter */
@@ -2548,21 +2571,31 @@ populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
if (ctx->dims[ndim] == -1)
ctx->dims[ndim] = dim; /* assign dimension if not yet known */
else if (ctx->dims[ndim] != dim)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed JSON array"),
errdetail("Multidimensional arrays must have "
"sub-arrays with matching dimensions.")));
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
+
/* reset the current array dimension size counter */
ctx->sizes[ndim] = 0;
/* increment the parent dimension counter if it is a nested sub-array */
if (ndim > 0)
ctx->sizes[ndim - 1]++;
+
+ return true;
}
-static void
+/*
+ * Returns true if the array element value was successfully extracted from jsv
+ * and added to ctx->astate. False if an error occurred when doing so.
+ */
+static bool
populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
{
Datum element;
@@ -2573,13 +2606,18 @@ populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
ctx->aio->element_type,
ctx->aio->element_typmod,
NULL, ctx->mcxt, PointerGetDatum(NULL),
- jsv, &element_isnull);
+ jsv, &element_isnull, ctx->escontext);
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
accumArrayResult(ctx->astate, element, element_isnull,
ctx->aio->element_type, ctx->acxt);
Assert(ndim > 0);
ctx->sizes[ndim - 1]++; /* increment current dimension counter */
+
+ return true;
}
/* json object start handler for populate_array_json() */
@@ -2592,7 +2630,12 @@ populate_array_object_start(void *_state)
if (state->ctx->ndims <= 0)
populate_array_assign_ndims(state->ctx, ndim);
else if (ndim < state->ctx->ndims)
+ {
populate_array_report_expected_array(state->ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2609,7 +2652,11 @@ populate_array_array_end(void *_state)
populate_array_assign_ndims(ctx, ndim + 1);
if (ndim < ctx->ndims)
- populate_array_check_dimension(ctx, ndim);
+ {
+ /* Report if an error occurred. */
+ if (!populate_array_check_dimension(ctx, ndim))
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2667,7 +2714,9 @@ populate_array_element_end(void *_state, bool isnull)
state->element_start) * sizeof(char);
}
- populate_array_element(ctx, ndim, &jsv);
+ /* Report if an error occurred. */
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return JSON_SEM_ACTION_FAILED;
}
return JSON_SUCCESS;
@@ -2684,7 +2733,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
if (ctx->ndims <= 0)
populate_array_assign_ndims(ctx, ndim);
else if (ndim < ctx->ndims)
+ {
populate_array_report_expected_array(ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
if (ndim == ctx->ndims)
{
@@ -2697,8 +2751,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
return JSON_SUCCESS;
}
-/* parse a json array and populate array */
-static void
+/*
+ * Parse a json array and populate array
+ *
+ * Returns false if an error occurs when parsing.
+ */
+static bool
populate_array_json(PopulateArrayContext *ctx, char *json, int len)
{
PopulateArrayState state;
@@ -2715,19 +2773,25 @@ populate_array_json(PopulateArrayContext *ctx, char *json, int len)
sem.array_element_end = populate_array_element_end;
sem.scalar = populate_array_scalar;
- pg_parse_json_or_ereport(state.lex, &sem);
-
- /* number of dimensions should be already known */
- Assert(ctx->ndims > 0 && ctx->dims);
+ if (pg_parse_json_or_errsave(state.lex, &sem, ctx->escontext))
+ {
+ /* number of dimensions should be already known */
+ Assert(ctx->ndims > 0 && ctx->dims);
+ }
pfree(state.lex);
+
+ return !SOFT_ERROR_OCCURRED(ctx->escontext);
}
/*
* populate_array_dim_jsonb() -- Iterate recursively through jsonb sub-array
* elements and accumulate result using given ArrayBuildState.
+ *
+ * Returns false if we return partway through because of an error in a
+ * subroutine.
*/
-static void
+static bool
populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
JsonbValue *jbv, /* jsonb sub-array */
int ndim) /* current dimension */
@@ -2741,7 +2805,12 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ {
populate_array_report_expected_array(ctx, ndim - 1);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
Assert(!JsonContainerIsScalar(jbc));
@@ -2762,7 +2831,10 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
(tok == WJB_ELEM &&
(val.type != jbvBinary ||
!JsonContainerIsArray(val.val.binary.data)))))
- populate_array_assign_ndims(ctx, ndim);
+ {
+ if (!populate_array_assign_ndims(ctx, ndim))
+ return false;
+ }
jsv.is_json = false;
jsv.val.jsonb = &val;
@@ -2775,16 +2847,21 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
* it is not the innermost dimension.
*/
if (ctx->ndims > 0 && ndim >= ctx->ndims)
- populate_array_element(ctx, ndim, &jsv);
+ {
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return false;
+ }
else
{
/* populate child sub-array */
- populate_array_dim_jsonb(ctx, &val, ndim + 1);
+ if (!populate_array_dim_jsonb(ctx, &val, ndim + 1))
+ return false;
/* number of dimensions should be already known */
Assert(ctx->ndims > 0 && ctx->dims);
- populate_array_check_dimension(ctx, ndim);
+ if (!populate_array_check_dimension(ctx, ndim))
+ return false;
}
tok = JsonbIteratorNext(&it, &val, true);
@@ -2795,14 +2872,22 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
/* free iterator, iterating until WJB_DONE */
tok = JsonbIteratorNext(&it, &val, true);
Assert(tok == WJB_DONE && !it);
+
+ return true;
}
-/* recursively populate an array from json/jsonb */
+/*
+ * Recursively populate an array from json/jsonb
+ *
+ * *isnull is set to true if an error is reported during parsing.
+ */
static Datum
populate_array(ArrayIOData *aio,
const char *colname,
MemoryContext mcxt,
- JsValue *jsv)
+ JsValue *jsv,
+ bool *isnull,
+ Node *escontext)
{
PopulateArrayContext ctx;
Datum result;
@@ -2817,14 +2902,27 @@ populate_array(ArrayIOData *aio,
ctx.ndims = 0; /* unknown yet */
ctx.dims = NULL;
ctx.sizes = NULL;
+ ctx.escontext = escontext;
if (jsv->is_json)
- populate_array_json(&ctx, jsv->val.json.str,
- jsv->val.json.len >= 0 ? jsv->val.json.len
- : strlen(jsv->val.json.str));
+ {
+ /* Return null if an error was found. */
+ if (!populate_array_json(&ctx, jsv->val.json.str,
+ jsv->val.json.len >= 0 ? jsv->val.json.len
+ : strlen(jsv->val.json.str)))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
+ }
else
{
- populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1);
+ /* Return null if an error was found. */
+ if (!populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
ctx.dims[0] = ctx.sizes[0];
}
@@ -2842,11 +2940,16 @@ populate_array(ArrayIOData *aio,
pfree(ctx.sizes);
pfree(lbs);
+ *isnull = false;
return result;
}
-static void
-JsValueToJsObject(JsValue *jsv, JsObject *jso)
+/*
+ * Returns false if an error occurs, provided escontext points to an
+ * ErrorSaveContext.
+ */
+static bool
+JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext)
{
jso->is_json = jsv->is_json;
@@ -2858,7 +2961,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
jsv->val.json.len >= 0
? jsv->val.json.len
: strlen(jsv->val.json.str),
- "populate_composite");
+ "populate_composite",
+ escontext);
}
else
{
@@ -2876,7 +2980,7 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
is_scalar = IsAJsonbScalar(jbv) ||
(jbv->type == jbvBinary &&
JsonContainerIsScalar(jbv->val.binary.data));
- ereport(ERROR,
+ errsave(escontext,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
is_scalar
? errmsg("cannot call %s on a scalar",
@@ -2885,6 +2989,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
"populate_composite")));
}
}
+
+ return !SOFT_ERROR_OCCURRED(escontext);
}
/* acquire or update cached tuple descriptor for a composite type */
@@ -2911,7 +3017,12 @@ update_cached_tupdesc(CompositeIOData *io, MemoryContext mcxt)
}
}
-/* recursively populate a composite (row type) value from json/jsonb */
+/*
+ * Recursively populate a composite (row type) value from json/jsonb
+ *
+ * Returns null if an error occurs in a subroutine, provided escontext points
+ * to an ErrorSaveContext.
+ */
static Datum
populate_composite(CompositeIOData *io,
Oid typid,
@@ -2919,14 +3030,15 @@ populate_composite(CompositeIOData *io,
MemoryContext mcxt,
HeapTupleHeader defaultval,
JsValue *jsv,
- bool isnull)
+ bool *isnull,
+ Node *escontext)
{
Datum result;
/* acquire/update cached tuple descriptor */
update_cached_tupdesc(io, mcxt);
- if (isnull)
+ if (*isnull)
result = (Datum) 0;
else
{
@@ -2934,11 +3046,15 @@ populate_composite(CompositeIOData *io,
JsObject jso;
/* prepare input value */
- JsValueToJsObject(jsv, &jso);
+ if (!JsValueToJsObject(jsv, &jso, escontext))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
/* populate resulting record tuple */
tuple = populate_record(io->tupdesc, &io->record_io,
- defaultval, mcxt, &jso);
+ defaultval, mcxt, &jso, escontext);
result = HeapTupleHeaderGetDatum(tuple);
JsObjectFree(&jso);
@@ -2950,14 +3066,20 @@ populate_composite(CompositeIOData *io,
* now, we can tell by comparing typid to base_typid.)
*/
if (typid != io->base_typid && typid != RECORDOID)
- domain_check(result, isnull, typid, &io->domain_info, mcxt);
+ domain_check(result, *isnull, typid, &io->domain_info, mcxt);
return result;
}
-/* populate non-null scalar value from json/jsonb value */
+/*
+ * Populate non-null scalar value from json/jsonb value.
+ *
+ * Returns null if an error occurs during the call to type input function,
+ * provided escontext is valid.
+ */
static Datum
-populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
+populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext)
{
Datum res;
char *str = NULL;
@@ -3028,7 +3150,12 @@ populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
elog(ERROR, "unrecognized jsonb type: %d", (int) jbv->type);
}
- res = InputFunctionCall(&io->typiofunc, str, io->typioparam, typmod);
+ if (!InputFunctionCallSafe(&io->typiofunc, str, io->typioparam, typmod,
+ escontext, &res))
+ {
+ res = (Datum) 0;
+ *isnull = true;
+ }
/* free temporary buffer */
if (str != json)
@@ -3043,7 +3170,8 @@ populate_domain(DomainIOData *io,
const char *colname,
MemoryContext mcxt,
JsValue *jsv,
- bool isnull)
+ bool isnull,
+ Node *escontext)
{
Datum res;
@@ -3054,8 +3182,8 @@ populate_domain(DomainIOData *io,
res = populate_record_field(io->base_io,
io->base_typid, io->base_typmod,
colname, mcxt, PointerGetDatum(NULL),
- jsv, &isnull);
- Assert(!isnull);
+ jsv, &isnull, escontext);
+ Assert(!isnull || SOFT_ERROR_OCCURRED(escontext));
}
domain_check(res, isnull, typid, &io->domain_info, mcxt);
@@ -3159,7 +3287,8 @@ populate_record_field(ColumnIOData *col,
MemoryContext mcxt,
Datum defaultval,
JsValue *jsv,
- bool *isnull)
+ bool *isnull,
+ Node *escontext)
{
TypeCat typcat;
@@ -3192,10 +3321,12 @@ populate_record_field(ColumnIOData *col,
switch (typcat)
{
case TYPECAT_SCALAR:
- return populate_scalar(&col->scalar_io, typid, typmod, jsv);
+ return populate_scalar(&col->scalar_io, typid, typmod, jsv,
+ isnull, escontext);
case TYPECAT_ARRAY:
- return populate_array(&col->io.array, colname, mcxt, jsv);
+ return populate_array(&col->io.array, colname, mcxt, jsv,
+ isnull, escontext);
case TYPECAT_COMPOSITE:
case TYPECAT_COMPOSITE_DOMAIN:
@@ -3204,11 +3335,12 @@ populate_record_field(ColumnIOData *col,
DatumGetPointer(defaultval)
? DatumGetHeapTupleHeader(defaultval)
: NULL,
- jsv, *isnull);
+ jsv, isnull,
+ escontext);
case TYPECAT_DOMAIN:
return populate_domain(&col->io.domain, typid, colname, mcxt,
- jsv, *isnull);
+ jsv, *isnull, escontext);
default:
elog(ERROR, "unrecognized type category '%c'", typcat);
@@ -3265,7 +3397,8 @@ populate_record(TupleDesc tupdesc,
RecordIOData **record_p,
HeapTupleHeader defaultval,
MemoryContext mcxt,
- JsObject *obj)
+ JsObject *obj,
+ Node *escontext)
{
RecordIOData *record = *record_p;
Datum *values;
@@ -3357,7 +3490,8 @@ populate_record(TupleDesc tupdesc,
mcxt,
nulls[i] ? (Datum) 0 : values[i],
&field,
- &nulls[i]);
+ &nulls[i],
+ escontext);
}
res = heap_form_tuple(tupdesc, values, nulls);
@@ -3444,6 +3578,7 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
JsValue jsv = {0};
HeapTupleHeader rec;
Datum rettuple;
+ bool isnull;
JsonbValue jbv;
MemoryContext fnmcxt = fcinfo->flinfo->fn_mcxt;
PopulateRecordCache *cache = fcinfo->flinfo->fn_extra;
@@ -3530,8 +3665,11 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
jbv.val.binary.len = VARSIZE(jb) - VARHDRSZ;
}
+ isnull = false;
rettuple = populate_composite(&cache->c.io.composite, cache->argtype,
- NULL, fnmcxt, rec, &jsv, false);
+ NULL, fnmcxt, rec, &jsv, &isnull,
+ NULL);
+ Assert(!isnull);
PG_RETURN_DATUM(rettuple);
}
@@ -3542,7 +3680,8 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
* decompose a json object into a hash table.
*/
static HTAB *
-get_json_object_as_hash(char *json, int len, const char *funcname)
+get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext)
{
HASHCTL ctl;
HTAB *tab;
@@ -3571,7 +3710,7 @@ get_json_object_as_hash(char *json, int len, const char *funcname)
sem->object_field_start = hash_object_field_start;
sem->object_field_end = hash_object_field_end;
- pg_parse_json_or_ereport(lex, sem);
+ pg_parse_json_or_errsave(lex, sem, escontext);
return tab;
}
@@ -3740,7 +3879,8 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
&cache->c.io.composite.record_io,
state->rec,
cache->fn_mcxt,
- obj);
+ obj,
+ NULL);
/* if it's domain over composite, check domain constraints */
if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
--
2.35.3
[application/octet-stream] v22-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch (2.3K, ../../CA+HiwqETtgH0ESDEEGDwTmazmtNeR7UsJS=BOoa=ghdYMo+Ufg@mail.gmail.com/3-v22-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch)
download | inline diff:
From a6540f6135b0a2e61d83479efb1d8e0b42bbf925 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:18 +0900
Subject: [PATCH v22 5/5] Claim SQL standard compliance for SQL/JSON features
Discussion: https://postgr.es/m/[email protected]
---
src/backend/catalog/sql_features.txt | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt
index b33065d7bf..b379c71df9 100644
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -548,15 +548,15 @@ T811 Basic SQL/JSON constructor functions YES
T812 SQL/JSON: JSON_OBJECTAGG YES
T813 SQL/JSON: JSON_ARRAYAGG with ORDER BY YES
T814 Colon in JSON_OBJECT or JSON_OBJECTAGG YES
-T821 Basic SQL/JSON query operators NO
+T821 Basic SQL/JSON query operators YES
T822 SQL/JSON: IS JSON WITH UNIQUE KEYS predicate YES
-T823 SQL/JSON: PASSING clause NO
-T824 JSON_TABLE: specific PLAN clause NO
-T825 SQL/JSON: ON EMPTY and ON ERROR clauses NO
-T826 General value expression in ON ERROR or ON EMPTY clauses NO
-T827 JSON_TABLE: sibling NESTED COLUMNS clauses NO
-T828 JSON_QUERY NO
-T829 JSON_QUERY: array wrapper options NO
+T823 SQL/JSON: PASSING clause YES
+T824 JSON_TABLE: specific PLAN clause YES
+T825 SQL/JSON: ON EMPTY and ON ERROR clauses YES
+T826 General value expression in ON ERROR or ON EMPTY clauses YES
+T827 JSON_TABLE: sibling NESTED COLUMNS clauses YES
+T828 JSON_QUERY YES
+T829 JSON_QUERY: array wrapper options YES
T830 Enforcing unique keys in SQL/JSON constructor functions YES
T831 SQL/JSON path language: strict mode YES
T832 SQL/JSON path language: item method YES
@@ -565,7 +565,7 @@ T834 SQL/JSON path language: wildcard member accessor YES
T835 SQL/JSON path language: filter expressions YES
T836 SQL/JSON path language: starts with predicate YES
T837 SQL/JSON path language: regex_like predicate YES
-T838 JSON_TABLE: PLAN DEFAULT clause NO
+T838 JSON_TABLE: PLAN DEFAULT clause YES
T839 Formatted cast of datetimes to/from character strings NO
T840 Hex integer literals in SQL/JSON path language YES
T851 SQL/JSON: optional keywords for default syntax YES
--
2.35.3
[application/octet-stream] v22-0001-Add-soft-error-handling-to-some-expression-nodes.patch (16.8K, ../../CA+HiwqETtgH0ESDEEGDwTmazmtNeR7UsJS=BOoa=ghdYMo+Ufg@mail.gmail.com/4-v22-0001-Add-soft-error-handling-to-some-expression-nodes.patch)
download | inline diff:
From 0efe0ca557f9428aba5f4f5a8ecf19ae27563f4b Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Fri, 29 Sep 2023 13:22:15 +0900
Subject: [PATCH v22 1/5] Add soft error handling to some expression nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This adjusts the CoerceViaIO and CoerceToDomain expression evaluation
code to handle errors softly.
For CoerceViaIo, this means using InputFunctionCallSafe(), which
provides the option to handle errors softly, instead of calling the
type input function directly.
For CoerceToDomain, this simply entails replacing the ereport() in
ExecEvalConstraintCheck() by errsave().
In both cases, the ErrorSaveContext to be used is populated in the
expression's struct in the ExprEvalStep. The ErrorSaveContext can be
passed by setting ExprState.escontext to point to it before calling
ExecInitExprRec() on the expression tree whose errors are to be
suppressed.
Note that no call site of ExecInitExprRec() has been changed in this
commit, so there's no functional change. This is intended for
implementing new SQL/JSON expression nodes in future commits that
will use to it suppress errors that may occur during type coercions.
Reviewed-by: Álvaro Herrera
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/executor/execExpr.c | 28 +++++------
src/backend/executor/execExprInterp.c | 36 ++++++--------
src/backend/jit/llvm/llvmjit.c | 4 ++
src/backend/jit/llvm/llvmjit_expr.c | 71 +++++++++++++++------------
src/backend/jit/llvm/llvmjit_types.c | 3 ++
src/include/executor/execExpr.h | 5 +-
src/include/jit/llvmjit.h | 2 +
src/include/jit/llvmjit_emit.h | 9 ++++
src/include/nodes/execnodes.h | 7 +++
9 files changed, 95 insertions(+), 70 deletions(-)
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 2c62b0c9c8..78796acd8f 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -139,6 +139,7 @@ ExecInitExpr(Expr *node, PlanState *parent)
state->expr = node;
state->parent = parent;
state->ext_params = NULL;
+ state->escontext = NULL;
/* Insert setup steps as needed */
ExecCreateExprSetupSteps(state, (Node *) node);
@@ -176,6 +177,7 @@ ExecInitExprWithParams(Expr *node, ParamListInfo ext_params)
state->expr = node;
state->parent = NULL;
state->ext_params = ext_params;
+ state->escontext = NULL;
/* Insert setup steps as needed */
ExecCreateExprSetupSteps(state, (Node *) node);
@@ -228,6 +230,7 @@ ExecInitQual(List *qual, PlanState *parent)
state->expr = (Expr *) qual;
state->parent = parent;
state->ext_params = NULL;
+ state->escontext = NULL;
/* mark expression as to be used with ExecQual() */
state->flags = EEO_FLAG_IS_QUAL;
@@ -373,6 +376,7 @@ ExecBuildProjectionInfo(List *targetList,
state->expr = (Expr *) targetList;
state->parent = parent;
state->ext_params = NULL;
+ state->escontext = NULL;
state->resultslot = slot;
@@ -544,6 +548,7 @@ ExecBuildUpdateProjection(List *targetList,
state->expr = NULL; /* not used */
state->parent = parent;
state->ext_params = NULL;
+ state->escontext = NULL;
state->resultslot = slot;
@@ -1549,8 +1554,6 @@ ExecInitExprRec(Expr *node, ExprState *state,
CoerceViaIO *iocoerce = (CoerceViaIO *) node;
Oid iofunc;
bool typisvarlena;
- Oid typioparam;
- FunctionCallInfo fcinfo_in;
/* evaluate argument into step's result area */
ExecInitExprRec(iocoerce->arg, state, resv, resnull);
@@ -1579,25 +1582,13 @@ ExecInitExprRec(Expr *node, ExprState *state,
/* lookup the result type's input function */
scratch.d.iocoerce.finfo_in = palloc0(sizeof(FmgrInfo));
- scratch.d.iocoerce.fcinfo_data_in = palloc0(SizeForFunctionCallInfo(3));
-
getTypeInputInfo(iocoerce->resulttype,
- &iofunc, &typioparam);
+ &iofunc, &scratch.d.iocoerce.typioparam);
fmgr_info(iofunc, scratch.d.iocoerce.finfo_in);
fmgr_info_set_expr((Node *) node, scratch.d.iocoerce.finfo_in);
- InitFunctionCallInfoData(*scratch.d.iocoerce.fcinfo_data_in,
- scratch.d.iocoerce.finfo_in,
- 3, InvalidOid, NULL, NULL);
- /*
- * We can preload the second and third arguments for the input
- * function, since they're constants.
- */
- fcinfo_in = scratch.d.iocoerce.fcinfo_data_in;
- fcinfo_in->args[1].value = ObjectIdGetDatum(typioparam);
- fcinfo_in->args[1].isnull = false;
- fcinfo_in->args[2].value = Int32GetDatum(-1);
- fcinfo_in->args[2].isnull = false;
+ /* Use the ErrorSaveContext passed by the caller. */
+ scratch.d.iocoerce.escontext = state->escontext;
ExprEvalPushStep(state, &scratch);
break;
@@ -1628,6 +1619,7 @@ ExecInitExprRec(Expr *node, ExprState *state,
elemstate->expr = acoerce->elemexpr;
elemstate->parent = state->parent;
elemstate->ext_params = state->ext_params;
+ state->escontext = NULL;
elemstate->innermost_caseval = (Datum *) palloc(sizeof(Datum));
elemstate->innermost_casenull = (bool *) palloc(sizeof(bool));
@@ -3306,6 +3298,8 @@ ExecInitCoerceToDomain(ExprEvalStep *scratch, CoerceToDomain *ctest,
/* we'll allocate workspace only if needed */
scratch->d.domaincheck.checkvalue = NULL;
scratch->d.domaincheck.checknull = NULL;
+ /* Use the ErrorSaveContext passed by the caller. */
+ scratch->d.domaincheck.escontext = state->escontext;
/*
* Evaluate argument - it's fine to directly store it into resv/resnull,
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 24c2b60c62..c8018da19f 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -63,6 +63,7 @@
#include "executor/nodeSubplan.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/miscnodes.h"
#include "nodes/nodeFuncs.h"
#include "parser/parsetree.h"
#include "pgstat.h"
@@ -1177,29 +1178,24 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
/* call input function (similar to InputFunctionCall) */
if (!op->d.iocoerce.finfo_in->fn_strict || str != NULL)
{
- FunctionCallInfo fcinfo_in;
- Datum d;
-
- fcinfo_in = op->d.iocoerce.fcinfo_data_in;
- fcinfo_in->args[0].value = PointerGetDatum(str);
- fcinfo_in->args[0].isnull = *op->resnull;
- /* second and third arguments are already set up */
-
- fcinfo_in->isnull = false;
- d = FunctionCallInvoke(fcinfo_in);
- *op->resvalue = d;
+ /*
+ * InputFunctionCallSafe() writes directly into *op->resvalue.
+ * Return NULL if an error is reported.
+ */
+ if (!InputFunctionCallSafe(op->d.iocoerce.finfo_in, str,
+ op->d.iocoerce.typioparam, -1,
+ (Node *) op->d.iocoerce.escontext,
+ op->resvalue))
+ *op->resnull = true;
- /* Should get null result if and only if str is NULL */
- if (str == NULL)
- {
+ /*
+ * Should get null result if and only if str is NULL or if we
+ * got an error above.
+ */
+ if (str == NULL || SOFT_ERROR_OCCURRED(state->escontext))
Assert(*op->resnull);
- Assert(fcinfo_in->isnull);
- }
else
- {
Assert(!*op->resnull);
- Assert(!fcinfo_in->isnull);
- }
}
EEO_NEXT();
@@ -3745,7 +3741,7 @@ ExecEvalConstraintCheck(ExprState *state, ExprEvalStep *op)
{
if (!*op->d.domaincheck.checknull &&
!DatumGetBool(*op->d.domaincheck.checkvalue))
- ereport(ERROR,
+ errsave((Node *) op->d.domaincheck.escontext,
(errcode(ERRCODE_CHECK_VIOLATION),
errmsg("value for domain %s violates check constraint \"%s\"",
format_type_be(op->d.domaincheck.resulttype),
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 4dfaf79743..125e1e73ae 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -70,12 +70,14 @@ LLVMTypeRef StructHeapTupleTableSlot;
LLVMTypeRef StructMinimalTupleTableSlot;
LLVMTypeRef StructMemoryContextData;
LLVMTypeRef StructFunctionCallInfoData;
+LLVMTypeRef StructFmgrInfo;
LLVMTypeRef StructExprContext;
LLVMTypeRef StructExprEvalStep;
LLVMTypeRef StructExprState;
LLVMTypeRef StructAggState;
LLVMTypeRef StructAggStatePerGroupData;
LLVMTypeRef StructAggStatePerTransData;
+LLVMTypeRef StructErrorSaveContext;
LLVMValueRef AttributeTemplate;
@@ -1118,6 +1120,7 @@ llvm_create_types(void)
StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
StructExprState = llvm_pg_var_type("StructExprState");
StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
+ StructFmgrInfo = llvm_pg_var_type("StructFmgrInfo");
StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
@@ -1127,6 +1130,7 @@ llvm_create_types(void)
StructAggState = llvm_pg_var_type("StructAggState");
StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
+ StructErrorSaveContext = llvm_pg_var_type("StructErrorSaveContext");
AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
}
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index 4b51aa1ce0..7d44a4c9f4 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1251,14 +1251,9 @@ llvm_compile_expr(ExprState *state)
case EEOP_IOCOERCE:
{
- FunctionCallInfo fcinfo_out,
- fcinfo_in;
- LLVMValueRef v_fn_out,
- v_fn_in;
- LLVMValueRef v_fcinfo_out,
- v_fcinfo_in;
- LLVMValueRef v_fcinfo_in_isnullp;
- LLVMValueRef v_retval;
+ FunctionCallInfo fcinfo_out;
+ LLVMValueRef v_fn_out;
+ LLVMValueRef v_fcinfo_out;
LLVMValueRef v_resvalue;
LLVMValueRef v_resnull;
@@ -1271,7 +1266,6 @@ llvm_compile_expr(ExprState *state)
LLVMBasicBlockRef b_inputcall;
fcinfo_out = op->d.iocoerce.fcinfo_data_out;
- fcinfo_in = op->d.iocoerce.fcinfo_data_in;
b_skipoutput = l_bb_before_v(opblocks[opno + 1],
"op.%d.skipoutputnull", opno);
@@ -1283,14 +1277,7 @@ llvm_compile_expr(ExprState *state)
"op.%d.inputcall", opno);
v_fn_out = llvm_function_reference(context, b, mod, fcinfo_out);
- v_fn_in = llvm_function_reference(context, b, mod, fcinfo_in);
v_fcinfo_out = l_ptr_const(fcinfo_out, l_ptr(StructFunctionCallInfoData));
- v_fcinfo_in = l_ptr_const(fcinfo_in, l_ptr(StructFunctionCallInfoData));
-
- v_fcinfo_in_isnullp =
- LLVMBuildStructGEP(b, v_fcinfo_in,
- FIELDNO_FUNCTIONCALLINFODATA_ISNULL,
- "v_fcinfo_in_isnull");
/* output functions are not called on nulls */
v_resnull = LLVMBuildLoad(b, v_resnullp, "");
@@ -1356,24 +1343,44 @@ llvm_compile_expr(ExprState *state)
LLVMBuildBr(b, b_inputcall);
}
+ /*
+ * Call the input function.
+ *
+ * If op->d.iocoerce.escontext references an
+ * ErrorSaveContext, InputFunctionCallSafe() would return
+ * false upon encountering an error.
+ */
LLVMPositionBuilderAtEnd(b, b_inputcall);
- /* set arguments */
- /* arg0: output */
- LLVMBuildStore(b, v_output,
- l_funcvaluep(b, v_fcinfo_in, 0));
- LLVMBuildStore(b, v_resnull,
- l_funcnullp(b, v_fcinfo_in, 0));
-
- /* arg1: ioparam: preset in execExpr.c */
- /* arg2: typmod: preset in execExpr.c */
-
- /* reset fcinfo_in->isnull */
- LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_in_isnullp);
- /* and call function */
- v_retval = LLVMBuildCall(b, v_fn_in, &v_fcinfo_in, 1,
- "funccall_iocoerce_in");
+ {
+ Oid ioparam = op->d.iocoerce.typioparam;
+ LLVMValueRef v_params[6];
+ LLVMValueRef v_success;
+
+ v_params[0] = l_ptr_const(op->d.iocoerce.finfo_in,
+ l_ptr(StructFmgrInfo));
+ v_params[1] = v_output;
+ v_params[2] = l_oid_const(lc, ioparam);
+ v_params[3] = l_int32_const(lc, -1);
+ v_params[4] = l_ptr_const(op->d.iocoerce.escontext,
+ l_ptr(StructErrorSaveContext));
- LLVMBuildStore(b, v_retval, v_resvaluep);
+ /*
+ * InputFunctionCallSafe() will write directly into
+ * *op->resvalue.
+ */
+ v_params[5] = v_resvaluep;
+
+ v_success = LLVMBuildCall(b, llvm_pg_func(mod, "InputFunctionCallSafe"),
+ v_params, lengthof(v_params),
+ "funccall_iocoerce_in_safe");
+
+ /*
+ * Return null if InputFunctionCallSafe() encountered
+ * an error.
+ */
+ v_resnullp = LLVMBuildICmp(b, LLVMIntEQ, v_success,
+ l_sbool_const(0), "");
+ }
LLVMBuildBr(b, opblocks[opno + 1]);
break;
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 41ac4c6f45..e1e9625038 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -59,6 +59,7 @@ AggStatePerTransData StructAggStatePerTransData;
ExprContext StructExprContext;
ExprEvalStep StructExprEvalStep;
ExprState StructExprState;
+FmgrInfo StructFmgrInfo;
FunctionCallInfoBaseData StructFunctionCallInfoData;
HeapTupleData StructHeapTupleData;
MemoryContextData StructMemoryContextData;
@@ -66,6 +67,7 @@ TupleTableSlot StructTupleTableSlot;
HeapTupleTableSlot StructHeapTupleTableSlot;
MinimalTupleTableSlot StructMinimalTupleTableSlot;
TupleDescData StructTupleDescData;
+ErrorSaveContext StructErrorSaveContext;
/*
@@ -136,6 +138,7 @@ void *referenced_functions[] =
ExecEvalJsonConstructor,
ExecEvalJsonIsPredicate,
MakeExpandedObjectReadOnlyInternal,
+ InputFunctionCallSafe,
slot_getmissingattrs,
slot_getsomeattrs_int,
strlen,
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index 048573c2bc..59f3b043c6 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -16,6 +16,7 @@
#include "executor/nodeAgg.h"
#include "nodes/execnodes.h"
+#include "nodes/miscnodes.h"
/* forward references to avoid circularity */
struct ExprEvalStep;
@@ -416,7 +417,8 @@ typedef struct ExprEvalStep
FunctionCallInfo fcinfo_data_out;
/* lookup and call info for result type's input function */
FmgrInfo *finfo_in;
- FunctionCallInfo fcinfo_data_in;
+ Oid typioparam;
+ ErrorSaveContext *escontext;
} iocoerce;
/* for EEOP_SQLVALUEFUNCTION */
@@ -547,6 +549,7 @@ typedef struct ExprEvalStep
bool *checknull;
/* OID of domain type */
Oid resulttype;
+ ErrorSaveContext *escontext;
} domaincheck;
/* for EEOP_CONVERT_ROWTYPE */
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 6d90a16f79..5b7681eba9 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -75,6 +75,7 @@ extern PGDLLIMPORT LLVMTypeRef StructTupleTableSlot;
extern PGDLLIMPORT LLVMTypeRef StructHeapTupleTableSlot;
extern PGDLLIMPORT LLVMTypeRef StructMinimalTupleTableSlot;
extern PGDLLIMPORT LLVMTypeRef StructMemoryContextData;
+extern PGDLLIMPORT LLVMTypeRef StructFmgrInfo;
extern PGDLLIMPORT LLVMTypeRef StructFunctionCallInfoData;
extern PGDLLIMPORT LLVMTypeRef StructExprContext;
extern PGDLLIMPORT LLVMTypeRef StructExprEvalStep;
@@ -82,6 +83,7 @@ extern PGDLLIMPORT LLVMTypeRef StructExprState;
extern PGDLLIMPORT LLVMTypeRef StructAggState;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerTransData;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerGroupData;
+extern PGDLLIMPORT LLVMTypeRef StructErrorSaveContext;
extern PGDLLIMPORT LLVMValueRef AttributeTemplate;
diff --git a/src/include/jit/llvmjit_emit.h b/src/include/jit/llvmjit_emit.h
index 5e74543be4..ead46a64ae 100644
--- a/src/include/jit/llvmjit_emit.h
+++ b/src/include/jit/llvmjit_emit.h
@@ -85,6 +85,15 @@ l_sizet_const(size_t i)
return LLVMConstInt(TypeSizeT, i, false);
}
+/*
+ * Emit constant oid.
+ */
+static inline LLVMValueRef
+l_oid_const(LLVMContextRef lc, Oid i)
+{
+ return LLVMConstInt(LLVMInt32TypeInContext(lc), i, false);
+}
+
/*
* Emit constant boolean, as used for storage (e.g. global vars, structs).
*/
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index cb714f4a19..61d97b30bc 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -34,6 +34,7 @@
#include "fmgr.h"
#include "lib/ilist.h"
#include "lib/pairingheap.h"
+#include "nodes/miscnodes.h"
#include "nodes/params.h"
#include "nodes/plannodes.h"
#include "nodes/tidbitmap.h"
@@ -129,6 +130,12 @@ typedef struct ExprState
Datum *innermost_domainval;
bool *innermost_domainnull;
+
+ /*
+ * For expression nodes that support soft errors. Should be set to NULL
+ * before calling ExecInitExprRec() if the caller wants errors thrown.
+ */
+ ErrorSaveContext *escontext;
} ExprState;
--
2.35.3
[application/octet-stream] v22-0004-JSON_TABLE.patch (170.9K, ../../CA+HiwqETtgH0ESDEEGDwTmazmtNeR7UsJS=BOoa=ghdYMo+Ufg@mail.gmail.com/5-v22-0004-JSON_TABLE.patch)
download | inline diff:
From 9574df49d247a5aaebc842bc3c9bf69abea7c61c Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:14 +0900
Subject: [PATCH v22 4/5] JSON_TABLE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This feature allows jsonb data to be treated as a table and thus
used in a FROM clause like other tabular data. Data can be selected
from the jsonb using jsonpath expressions, and hoisted out of nested
structures in the jsonb to form multiple rows, more or less like an
outer join.
This also adds the PLAN clauses for JSON_TABLE, which allow the user
to specify how data from nested paths are joined, allowing
considerable freedom in shaping the tabular output of JSON_TABLE.
PLAN DEFAULT allows the user to specify the global strategies when
dealing with sibling or child nested paths. The is often sufficient
to achieve the necessary goal, and is considerably simpler than the
full PLAN clause, which allows the user to specify the strategy to be
used for each named nested path.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 496 ++++++++
src/backend/commands/explain.c | 8 +-
src/backend/executor/execExprInterp.c | 10 +
src/backend/executor/nodeTableFuncscan.c | 27 +-
src/backend/nodes/makefuncs.c | 19 +
src/backend/nodes/nodeFuncs.c | 30 +
src/backend/parser/Makefile | 1 +
src/backend/parser/gram.y | 476 +++++++-
src/backend/parser/meson.build | 1 +
src/backend/parser/parse_clause.c | 14 +-
src/backend/parser/parse_expr.c | 13 +
src/backend/parser/parse_jsontable.c | 751 ++++++++++++
src/backend/parser/parse_relation.c | 5 +-
src/backend/parser/parse_target.c | 3 +
src/backend/utils/adt/jsonpath_exec.c | 543 +++++++++
src/backend/utils/adt/ruleutils.c | 279 ++++-
src/include/nodes/execnodes.h | 2 +
src/include/nodes/makefuncs.h | 2 +
src/include/nodes/parsenodes.h | 90 ++
src/include/nodes/primnodes.h | 61 +-
src/include/parser/kwlist.h | 4 +
src/include/parser/parse_clause.h | 3 +
src/include/utils/jsonpath.h | 3 +
src/test/regress/expected/json_sqljson.out | 6 +
src/test/regress/expected/jsonb_sqljson.out | 1186 +++++++++++++++++++
src/test/regress/sql/json_sqljson.sql | 4 +
src/test/regress/sql/jsonb_sqljson.sql | 674 +++++++++++
src/tools/pgindent/typedefs.list | 13 +
28 files changed, 4697 insertions(+), 27 deletions(-)
create mode 100644 src/backend/parser/parse_jsontable.c
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index ddc4f4f6aa..adfe01f23e 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17206,6 +17206,502 @@ array w/o UK? | t
</sect2>
+ <sect2 id="functions-sqljson-table">
+ <title>JSON_TABLE</title>
+ <indexterm>
+ <primary>json_table</primary>
+ </indexterm>
+
+ <para>
+ <function>json_table</function> is an SQL/JSON function which
+ queries <acronym>JSON</acronym> data
+ and presents the results as a relational view, which can be accessed as a
+ regular SQL table. You can only use <function>json_table</function> inside the
+ <literal>FROM</literal> clause of a <literal>SELECT</literal> statement.
+ </para>
+
+ <para>
+ Taking JSON data as input, <function>json_table</function> uses
+ a path expression to extract a part of the provided data that
+ will be used as a <firstterm>row pattern</firstterm> for the
+ constructed view. Each SQL/JSON item at the top level of the row pattern serves
+ as the source for a separate row in the constructed relational view.
+ </para>
+
+ <para>
+ To split the row pattern into columns, <function>json_table</function>
+ provides the <literal>COLUMNS</literal> clause that defines the
+ schema of the created view. For each column to be constructed,
+ this clause provides a separate path expression that evaluates
+ the row pattern, extracts a JSON item, and returns it as a
+ separate SQL value for the specified column. If the required value
+ is stored in a nested level of the row pattern, it can be extracted
+ using the <literal>NESTED PATH</literal> subclause. Joining the
+ columns returned by <literal>NESTED PATH</literal> can add multiple
+ new rows to the constructed view. Such rows are called
+ <firstterm>child rows</firstterm>, as opposed to the <firstterm>parent row</firstterm>
+ that generates them.
+ </para>
+
+ <para>
+ The rows produced by <function>JSON_TABLE</function> are laterally
+ joined to the row that generated them, so you do not have to explicitly join
+ the constructed view with the original table holding <acronym>JSON</acronym>
+ data. Optionally, you can specify how to join the columns returned
+ by <literal>NESTED PATH</literal> using the <literal>PLAN</literal> clause.
+ </para>
+
+ <para>
+ Each <literal>NESTED PATH</literal> clause can generate one or more
+ columns. Columns produced by <literal>NESTED PATH</literal>s at the
+ same level are considered to be <firstterm>siblings</firstterm>,
+ while a column produced by a <literal>NESTED PATH</literal> is
+ considered to be a child of the column produced by a
+ <literal>NESTED PATH</literal> or row expression at a higher level.
+ Sibling columns are always joined first. Once they are processed,
+ the resulting rows are joined to the parent row.
+ </para>
+
+ <para>
+ The syntax is:
+ </para>
+
+<synopsis>
+JSON_TABLE (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> AS <replaceable>json_path_name</replaceable> </optional> <optional> PASSING { <replaceable>value</replaceable> AS <replaceable>varname</replaceable> } <optional>, ...</optional> </optional>
+ COLUMNS ( <replaceable class="parameter">json_table_column</replaceable> <optional>, ...</optional> )
+ <optional> { <literal>ERROR</literal> | <literal>EMPTY</literal> } <literal>ON ERROR</literal> </optional>
+ <optional>
+ PLAN ( <replaceable class="parameter">json_table_plan</replaceable> ) |
+ PLAN DEFAULT ( { INNER | OUTER } <optional> , { CROSS | UNION } </optional>
+ | { CROSS | UNION } <optional> , { INNER | OUTER } </optional> )
+ </optional>
+)
+
+<phrase>
+where <replaceable class="parameter">json_table_column</replaceable> is:
+</phrase>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> FORMAT <replaceable>json_representation</replaceable>
+ <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> EXISTS <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { ERROR | TRUE | FALSE | UNKNOWN } ON ERROR </optional>
+ | NESTED PATH <replaceable>json_path_specification</replaceable> <optional> AS <replaceable>path_name</replaceable> </optional>
+ COLUMNS ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ | <replaceable>name</replaceable> FOR ORDINALITY
+<phrase>
+<replaceable>json_table_plan</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> <optional> { OUTER | INNER } <replaceable>json_table_plan_primary</replaceable> </optional>
+ | <replaceable>json_table_plan_primary</replaceable> { UNION <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+ | <replaceable>json_table_plan_primary</replaceable> { CROSS <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+<phrase>
+<replaceable>json_table_plan_primary</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> | ( <replaceable>json_table_plan</replaceable> )
+</synopsis>
+
+ <para>
+ Each syntax element is described below in more detail.
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional></literal>
+ </term>
+ <listitem>
+ <para>
+ The input data to query, the JSON path expression defining the query,
+ and an optional <literal>PASSING</literal> clause, which can provide data
+ values to the <replaceable>path_expression</replaceable>.
+ The result of the input data
+ evaluation is called the <firstterm>row pattern</firstterm>. The row
+ pattern is used as the source for row values in the constructed view.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>COLUMNS</literal>( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ The <literal>COLUMNS</literal> clause defining the schema of the
+ constructed view. In this clause, you must specify all the columns
+ to be filled with SQL/JSON items.
+ The <replaceable>json_table_column</replaceable>
+ expression has the following syntax variants:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>name</replaceable> <replaceable>type</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional></literal>
+ </term>
+ <listitem>
+
+ <para>
+ Inserts a single SQL/JSON item into each row of
+ the specified column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON EMPTY</literal> and
+ <literal>ON ERROR</literal> clauses to define how to handle missing values
+ or structural errors.
+ <literal>WRAPPER</literal> and <literal>QUOTES</literal> clauses can only
+ be used with JSON, array, and composite types.
+ These clauses have the same syntax and semantics as for
+ <function>json_value</function> and <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <literal>FORMAT</literal> <replaceable>json_representation</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a composite SQL/JSON
+ item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>WRAPPER</literal>, <literal>QUOTES</literal>,
+ <literal>ON EMPTY</literal> and <literal>ON ERROR</literal> clauses
+ to define additional settings for the returned SQL/JSON items.
+ These clauses have the same syntax and semantics as
+ for <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable>
+ <literal>EXISTS</literal> <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a boolean item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated,
+ a check whether any SQL/JSON items were returned is done, and
+ the column is filled with the resulting boolean value, one for each row.
+ The specified <parameter>type</parameter> should have a cast from the
+ <type>boolean</type>.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON ERROR</literal> clause to define
+ error behavior. This clause has the same syntax and semantics as
+ for <function>json_exists</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>NESTED PATH</literal> <replaceable>json_path_specification</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional>
+ <literal>COLUMNS</literal> ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ Extracts SQL/JSON items from nested levels of the row pattern,
+ generates one or more columns as defined by the <literal>COLUMNS</literal>
+ subclause, and inserts the extracted SQL/JSON items into each row of these columns.
+ The <replaceable>json_table_column</replaceable> expression in the
+ <literal>COLUMNS</literal> subclause uses the same syntax as in the
+ parent <literal>COLUMNS</literal> clause.
+ </para>
+
+ <para>
+ The <literal>NESTED PATH</literal> syntax is recursive,
+ so you can go down multiple nested levels by specifying several
+ <literal>NESTED PATH</literal> subclauses within each other.
+ It allows to unnest the hierarchy of JSON objects and arrays
+ in a single function invocation rather than chaining several
+ <function>JSON_TABLE</function> expressions in an SQL statement.
+ </para>
+
+ <para>
+ You can use the <literal>PLAN</literal> clause to define how
+ to join the columns returned by <literal>NESTED PATH</literal> clauses.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <literal>FOR ORDINALITY</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Adds an ordinality column that provides sequential row numbering.
+ You can have only one ordinality column per table. Row numbering
+ is 1-based. For child rows that result from the <literal>NESTED PATH</literal>
+ clauses, the parent row number is repeated.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>AS</literal> <replaceable>json_path_name</replaceable>
+ </term>
+ <listitem>
+
+ <para>
+ The optional <replaceable>json_path_name</replaceable> serves as an
+ identifier of the provided <replaceable>json_path_specification</replaceable>.
+ The path name must be unique and distinct from the column names.
+ When using the <literal>PLAN</literal> clause, you must specify the names
+ for all the paths, including the row pattern. Each path name can appear in
+ the <literal>PLAN</literal> clause only once.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN</literal> ( <replaceable>json_table_plan</replaceable> )
+ </term>
+ <listitem>
+
+ <para>
+ Defines how to join the data returned by <literal>NESTED PATH</literal>
+ clauses to the constructed view.
+ </para>
+ <para>
+ To join columns with parent/child relationship, you can use:
+ </para>
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>INNER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>INNER JOIN</literal>, so that the parent row
+ is omitted from the output if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>OUTER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>LEFT OUTER JOIN</literal>, so that the parent row
+ is always included into the output even if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>, with NULL values
+ inserted into the child columns if the corresponding
+ values are missing.
+ </para>
+ <para>
+ This is the default option for joining columns with parent/child relationship.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>
+ To join sibling columns, you can use:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>UNION</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each value produced by each of the sibling
+ columns. The columns from the other siblings are set to null.
+ </para>
+ <para>
+ This is the default option for joining sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>CROSS</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each combination of values from the sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN DEFAULT</literal> ( <literal><replaceable>OUTER | INNER</replaceable> <optional>, <replaceable>UNION | CROSS</replaceable> </optional></literal> )
+ </term>
+ <listitem>
+ <para>
+ The terms can also be specified in reverse order. The
+ <literal>INNER</literal> or <literal>OUTER</literal> option defines the
+ joining plan for parent/child columns, while <literal>UNION</literal> or
+ <literal>CROSS</literal> affects joins of sibling columns. This form
+ of <literal>PLAN</literal> overrides the default plan for
+ all columns at once. Even though the path names are not included in the
+ <literal>PLAN DEFAULT</literal> form, to conform to the SQL/JSON standard
+ they must be provided for all the paths if the <literal>PLAN</literal>
+ clause is used.
+ </para>
+ <para>
+ <literal>PLAN DEFAULT</literal> is simpler than specifying a complete
+ <literal>PLAN</literal>, and is often all that is required to get the desired
+ output.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>Examples</para>
+
+ <para>
+ In these examples the following small table storing some JSON data will be used:
+<programlisting>
+CREATE TABLE my_films ( js jsonb );
+
+INSERT INTO my_films VALUES (
+'{ "favorites" : [
+ { "kind" : "comedy", "films" : [
+ { "title" : "Bananas",
+ "director" : "Woody Allen"},
+ { "title" : "The Dinner Game",
+ "director" : "Francis Veber" } ] },
+ { "kind" : "horror", "films" : [
+ { "title" : "Psycho",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "thriller", "films" : [
+ { "title" : "Vertigo",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "drama", "films" : [
+ { "title" : "Yojimbo",
+ "director" : "Akira Kurosawa" } ] }
+ ] }');
+</programlisting>
+ </para>
+ <para>
+ Query the <structname>my_films</structname> table holding
+ some JSON data about the films and create a view that
+ distributes the film genre, title, and director between separate columns:
+<screen>
+SELECT jt.* FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites[*]' COLUMNS (
+ id FOR ORDINALITY,
+ kind text PATH '$.kind',
+ NESTED PATH '$.films[*]' COLUMNS (
+ title text PATH '$.title',
+ director text PATH '$.director'))) AS jt;
+----+----------+------------------+-------------------
+ id | kind | title | director
+----+----------+------------------+-------------------
+ 1 | comedy | Bananas | Woody Allen
+ 1 | comedy | The Dinner Game | Francis Veber
+ 2 | horror | Psycho | Alfred Hitchcock
+ 3 | thriller | Vertigo | Alfred Hitchcock
+ 4 | drama | Yojimbo | Akira Kurosawa
+ (5 rows)
+</screen>
+ </para>
+
+ <para>
+ Find a director that has done films in two different genres:
+<screen>
+SELECT
+ director1 AS director, title1, kind1, title2, kind2
+FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites' AS favs COLUMNS (
+ NESTED PATH '$[*]' AS films1 COLUMNS (
+ kind1 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film1 COLUMNS (
+ title1 text PATH '$.title',
+ director1 text PATH '$.director')
+ ),
+ NESTED PATH '$[*]' AS films2 COLUMNS (
+ kind2 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film2 COLUMNS (
+ title2 text PATH '$.title',
+ director2 text PATH '$.director'
+ )
+ )
+ )
+ PLAN (favs OUTER ((films1 INNER film1) CROSS (films2 INNER film2)))
+ ) AS jt
+ WHERE kind1 > kind2 AND director1 = director2;
+
+ director | title1 | kind1 | title2 | kind2
+------------------+---------+----------+--------+--------
+ Alfred Hitchcock | Vertigo | thriller | Psycho | horror
+(1 row)
+</screen>
+ </para>
+ </sect2>
+
<sect2 id="functions-sqljson-path">
<title>The SQL/JSON Path Language</title>
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 13217807ee..a1b0328d1d 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -3870,7 +3870,13 @@ ExplainTargetRel(Plan *plan, Index rti, ExplainState *es)
break;
case T_TableFuncScan:
Assert(rte->rtekind == RTE_TABLEFUNC);
- objectname = "xmltable";
+ if (rte->tablefunc)
+ if (rte->tablefunc->functype == TFT_XMLTABLE)
+ objectname = "xmltable";
+ else /* Must be TFT_JSON_TABLE */
+ objectname = "json_table";
+ else
+ objectname = NULL;
objecttag = "Table Function Name";
break;
case T_ValuesScan:
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 9adf31682c..123121a91c 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -4310,6 +4310,11 @@ ExecEvalJsonExpr(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
break;
}
+ case JSON_TABLE_OP:
+ res = item;
+ resnull = false;
+ break;
+
default:
elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
*op->resnull = true;
@@ -4486,6 +4491,11 @@ ExecEvalJsonExprCoercion(ExprState *state, ExprEvalStep *op,
return jcstate->jump_eval_expr;
break;
+ case JSON_TABLE_OP:
+ /* No coercion needed. */
+ post_eval->coercion_done = true;
+ return op->d.jsonexpr_coercion.jump_coercion_done;
+
default:
elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
break;
diff --git a/src/backend/executor/nodeTableFuncscan.c b/src/backend/executor/nodeTableFuncscan.c
index a60dcd4943..0d7f518afd 100644
--- a/src/backend/executor/nodeTableFuncscan.c
+++ b/src/backend/executor/nodeTableFuncscan.c
@@ -28,6 +28,7 @@
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "utils/builtins.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/xml.h"
@@ -161,8 +162,9 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
scanstate->ss.ps.qual =
ExecInitQual(node->scan.plan.qual, &scanstate->ss.ps);
- /* Only XMLTABLE is supported currently */
- scanstate->routine = &XmlTableRoutine;
+ /* Only XMLTABLE and JSON_TABLE are supported currently */
+ scanstate->routine =
+ tf->functype == TFT_XMLTABLE ? &XmlTableRoutine : &JsonbTableRoutine;
scanstate->perTableCxt =
AllocSetContextCreate(CurrentMemoryContext,
@@ -182,6 +184,10 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
ExecInitExprList(tf->colexprs, (PlanState *) scanstate);
scanstate->coldefexprs =
ExecInitExprList(tf->coldefexprs, (PlanState *) scanstate);
+ scanstate->colvalexprs =
+ ExecInitExprList(tf->colvalexprs, (PlanState *) scanstate);
+ scanstate->passingvalexprs =
+ ExecInitExprList(tf->passingvalexprs, (PlanState *) scanstate);
scanstate->notnulls = tf->notnulls;
@@ -369,14 +375,17 @@ tfuncInitialize(TableFuncScanState *tstate, ExprContext *econtext, Datum doc)
routine->SetNamespace(tstate, ns_name, ns_uri);
}
- /* Install the row filter expression into the table builder context */
- value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
- if (isnull)
- ereport(ERROR,
- (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
- errmsg("row filter expression must not be null")));
+ if (routine->SetRowFilter)
+ {
+ /* Install the row filter expression into the table builder context */
+ value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
+ if (isnull)
+ ereport(ERROR,
+ (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
+ errmsg("row filter expression must not be null")));
- routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ }
/*
* Install the column filter expressions into the table builder context.
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index e1f7fde2bd..1436b9b5f6 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -876,6 +876,25 @@ makeJsonBehavior(JsonBehaviorType type, Node *default_expr, int location)
return behavior;
}
+/*
+ * makeJsonTableJoinedPlan -
+ * creates a joined JsonTablePlan node
+ */
+Node *
+makeJsonTableJoinedPlan(JsonTablePlanJoinType type, Node *plan1, Node *plan2,
+ int location)
+{
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_JOINED;
+ n->join_type = type;
+ n->plan1 = castNode(JsonTablePlan, plan1);
+ n->plan2 = castNode(JsonTablePlan, plan2);
+ n->location = location;
+
+ return (Node *) n;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index ef08ef2cbe..ca1747a2dd 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -2614,6 +2614,10 @@ expression_tree_walker_impl(Node *node,
return true;
if (WALK(tf->coldefexprs))
return true;
+ if (WALK(tf->colvalexprs))
+ return true;
+ if (WALK(tf->passingvalexprs))
+ return true;
}
break;
default:
@@ -3664,6 +3668,8 @@ expression_tree_mutator_impl(Node *node,
MUTATE(newnode->rowexpr, tf->rowexpr, Node *);
MUTATE(newnode->colexprs, tf->colexprs, List *);
MUTATE(newnode->coldefexprs, tf->coldefexprs, List *);
+ MUTATE(newnode->colvalexprs, tf->colvalexprs, List *);
+ MUTATE(newnode->passingvalexprs, tf->passingvalexprs, List *);
return (Node *) newnode;
}
break;
@@ -4095,6 +4101,30 @@ raw_expression_tree_walker_impl(Node *node,
return true;
}
break;
+ case T_JsonTable:
+ {
+ JsonTable *jt = (JsonTable *) node;
+
+ if (WALK(jt->common))
+ return true;
+ if (WALK(jt->columns))
+ return true;
+ }
+ break;
+ case T_JsonTableColumn:
+ {
+ JsonTableColumn *jtc = (JsonTableColumn *) node;
+
+ if (WALK(jtc->typeName))
+ return true;
+ if (WALK(jtc->on_empty))
+ return true;
+ if (WALK(jtc->on_error))
+ return true;
+ if (jtc->coltype == JTC_NESTED && WALK(jtc->columns))
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/parser/Makefile b/src/backend/parser/Makefile
index 9f1c4022bb..f4c0cc7f10 100644
--- a/src/backend/parser/Makefile
+++ b/src/backend/parser/Makefile
@@ -23,6 +23,7 @@ OBJS = \
parse_enr.o \
parse_expr.o \
parse_func.o \
+ parse_jsontable.o \
parse_merge.o \
parse_node.o \
parse_oper.o \
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index ffa8bbe770..15d9bd8425 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -654,19 +654,43 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_aggregate_func
json_api_common_syntax
json_argument
+ json_table
+ json_table_column_definition
+ json_table_ordinality_column_definition
+ json_table_regular_column_definition
+ json_table_formatted_column_definition
+ json_table_exists_column_definition
+ json_table_nested_columns
+ json_table_plan_clause_opt
+ json_table_plan
+ json_table_plan_simple
+ json_table_plan_parent_child
+ json_table_plan_outer
+ json_table_plan_inner
+ json_table_plan_sibling
+ json_table_plan_union
+ json_table_plan_cross
+ json_table_plan_primary
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
json_arguments
+ json_table_columns_clause
+ json_table_column_definition_list
+%type <str> json_table_column_path_specification_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
+ json_table_default_plan_choices
+ json_table_default_plan_inner_outer
+ json_table_default_plan_union_cross
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
%type <jsbehavior> json_value_behavior
json_query_behavior
json_exists_behavior
+ json_table_behavior
%type <js_quotes> json_quotes_clause_opt
@@ -733,7 +757,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
- JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_TABLE JSON_VALUE
KEY KEYS KEEP
@@ -744,8 +768,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
- NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
- NORMALIZE NORMALIZED
+ NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
+ NONE NORMALIZE NORMALIZED
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
@@ -753,8 +777,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PATH
+ PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -862,6 +886,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
* Using the same precedence as IDENT seems right for the reasons given above.
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
+%nonassoc COLUMNS
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
@@ -884,6 +909,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%left JOIN CROSS LEFT FULL RIGHT INNER_P NATURAL
+%nonassoc json_table_column
+%nonassoc NESTED
+%left PATH
%%
/*
@@ -13373,6 +13401,21 @@ table_ref: relation_expr opt_alias_clause
$2->alias = $4;
$$ = (Node *) $2;
}
+ | json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $1);
+
+ jt->alias = $2;
+ $$ = (Node *) jt;
+ }
+ | LATERAL_P json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $2);
+
+ jt->alias = $3;
+ jt->lateral = true;
+ $$ = (Node *) jt;
+ }
;
@@ -13940,6 +13983,8 @@ xmltable_column_option_el:
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(true), @1); }
| NULL_P
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(false), @1); }
+ | PATH b_expr
+ { $$ = makeDefElem("path", $2, @1); }
;
xml_namespace_list:
@@ -16747,6 +16792,11 @@ json_value_behavior:
| DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
;
+json_table_behavior:
+ ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
+ ;
+
/* ARRAY is a noise word */
json_wrapper_behavior:
WITHOUT WRAPPER { $$ = JSW_NONE; }
@@ -16768,6 +16818,414 @@ json_quotes_clause_opt:
| /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
;
+json_table:
+ JSON_TABLE '('
+ json_api_common_syntax
+ json_table_columns_clause
+ json_table_plan_clause_opt
+ ')'
+ {
+ JsonTable *n = makeNode(JsonTable);
+
+ n->common = (JsonCommon *) $3;
+ n->columns = $4;
+ n->plan = (JsonTablePlan *) $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_TABLE '('
+ json_api_common_syntax
+ json_table_columns_clause
+ json_table_plan_clause_opt
+ json_table_behavior ON ERROR_P
+ ')'
+ {
+ JsonTable *n = makeNode(JsonTable);
+
+ n->common = (JsonCommon *) $3;
+ n->columns = $4;
+ n->plan = (JsonTablePlan *) $5;
+ n->on_error = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_columns_clause:
+ COLUMNS '(' json_table_column_definition_list ')' { $$ = $3; }
+ ;
+
+json_table_column_definition_list:
+ json_table_column_definition
+ { $$ = list_make1($1); }
+ | json_table_column_definition_list ',' json_table_column_definition
+ { $$ = lappend($1, $3); }
+ ;
+
+json_table_column_definition:
+ json_table_ordinality_column_definition %prec json_table_column
+ | json_table_regular_column_definition %prec json_table_column
+ | json_table_formatted_column_definition %prec json_table_column
+ | json_table_exists_column_definition %prec json_table_column
+ | json_table_nested_columns
+ ;
+
+json_table_ordinality_column_definition:
+ ColId FOR ORDINALITY
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FOR_ORDINALITY;
+ n->name = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_regular_column_definition:
+ ColId Typename
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_REGULAR;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = $4; /* disallowed during parse-analysis! */
+ n->quotes = $5; /* disallowed during parse-analysis! */
+ n->pathspec = $3;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_value_behavior ON EMPTY_P
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_REGULAR;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = $4; /* disallowed during parse-analysis! */
+ n->quotes = $5; /* disallowed during parse-analysis! */
+ n->pathspec = $3;
+ n->on_empty = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_value_behavior ON ERROR_P
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_REGULAR;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = $4; /* disallowed during parse-analysis! */
+ n->quotes = $5; /* disallowed during parse-analysis! */
+ n->pathspec = $3;
+ n->on_error = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_value_behavior ON EMPTY_P
+ json_value_behavior ON ERROR_P
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_REGULAR;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = $4; /* disallowed during parse-analysis! */
+ n->quotes = $5; /* disallowed during parse-analysis! */
+ n->pathspec = $3;
+ n->on_empty = $6;
+ n->on_error = $9;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_formatted_column_definition:
+ ColId Typename FORMAT_LA JSON json_encoding_clause_opt
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FORMATTED;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_JSON, $5, @1);
+ n->pathspec = $6;
+ n->wrapper = $7;
+ if (n->wrapper != JSW_NONE && $8 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@7)));
+ n->quotes = $8;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename FORMAT_LA JSON json_encoding_clause_opt
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_query_behavior ON EMPTY_P
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FORMATTED;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_JSON, $5, @1);
+ n->pathspec = $6;
+ n->wrapper = $7;
+ if (n->wrapper != JSW_NONE && $8 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@7)));
+ n->quotes = $8;
+ n->on_empty = $9;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename FORMAT_LA JSON json_encoding_clause_opt
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_query_behavior ON ERROR_P
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FORMATTED;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_JSON, $5, @1);
+ n->pathspec = $6;
+ n->wrapper = $7;
+ if (n->wrapper != JSW_NONE && $8 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@7)));
+ n->quotes = $8;
+ n->on_error = $9;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename FORMAT_LA JSON json_encoding_clause_opt
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_query_behavior ON EMPTY_P
+ json_query_behavior ON ERROR_P
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FORMATTED;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_JSON, $5, @1);
+ n->pathspec = $6;
+ n->wrapper = $7;
+ if (n->wrapper != JSW_NONE && $8 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@7)));
+ n->quotes = $8;
+ n->on_empty = $9;
+ n->on_error = $12;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_exists_column_definition:
+ ColId Typename
+ EXISTS json_table_column_path_specification_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_EXISTS;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = JSW_NONE;
+ n->quotes = JS_QUOTES_UNSPEC;
+ n->pathspec = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename
+ EXISTS json_table_column_path_specification_clause_opt
+ json_exists_behavior ON ERROR_P
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_EXISTS;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = JSW_NONE;
+ n->quotes = JS_QUOTES_UNSPEC;
+ n->pathspec = $4;
+ n->on_error = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_column_path_specification_clause_opt:
+ PATH Sconst { $$ = $2; }
+ | /* EMPTY */ %prec json_table_column { $$ = NULL; }
+ ;
+
+json_table_nested_columns:
+ NESTED path_opt Sconst
+ json_table_columns_clause
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->columns = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | NESTED path_opt Sconst AS name
+ json_table_columns_clause
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->columns = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+path_opt:
+ PATH { }
+ | /* EMPTY */ { }
+ ;
+
+json_table_plan_clause_opt:
+ PLAN '(' json_table_plan ')' { $$ = $3; }
+ | PLAN DEFAULT '(' json_table_default_plan_choices ')'
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_DEFAULT;
+ n->join_type = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | /* EMPTY */ { $$ = NULL; }
+ ;
+
+json_table_plan:
+ json_table_plan_simple
+ | json_table_plan_parent_child
+ | json_table_plan_sibling
+ ;
+
+json_table_plan_simple:
+ name
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_SIMPLE;
+ n->pathname = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_plan_primary:
+ json_table_plan_simple { $$ = $1; }
+ | '(' json_table_plan ')'
+ {
+ castNode(JsonTablePlan, $2)->location = @1;
+ $$ = $2;
+ }
+ ;
+
+json_table_plan_outer:
+ json_table_plan_simple OUTER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_OUTER, $1, $3, @1); }
+ ;
+
+json_table_plan_inner:
+ json_table_plan_simple INNER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_INNER, $1, $3, @1); }
+ ;
+
+json_table_plan_parent_child:
+ json_table_plan_outer
+ | json_table_plan_inner
+ ;
+
+json_table_plan_union:
+ json_table_plan_primary UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ | json_table_plan_union UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ ;
+
+json_table_plan_cross:
+ json_table_plan_primary CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ | json_table_plan_cross CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ ;
+
+json_table_plan_sibling:
+ json_table_plan_union
+ | json_table_plan_cross
+ ;
+
+json_table_default_plan_choices:
+ json_table_default_plan_inner_outer { $$ = $1 | JSTPJ_UNION; }
+ | json_table_default_plan_union_cross { $$ = $1 | JSTPJ_OUTER; }
+ | json_table_default_plan_inner_outer ','
+ json_table_default_plan_union_cross { $$ = $1 | $3; }
+ | json_table_default_plan_union_cross ','
+ json_table_default_plan_inner_outer { $$ = $1 | $3; }
+ ;
+
+json_table_default_plan_inner_outer:
+ INNER_P { $$ = JSTPJ_INNER; }
+ | OUTER_P { $$ = JSTPJ_OUTER; }
+ ;
+
+json_table_default_plan_union_cross:
+ UNION { $$ = JSTPJ_UNION; }
+ | CROSS { $$ = JSTPJ_CROSS; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -17492,6 +17950,7 @@ unreserved_keyword:
| MOVE
| NAME_P
| NAMES
+ | NESTED
| NEW
| NEXT
| NFC
@@ -17526,6 +17985,8 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
+ | PLAN
| PLANS
| POLICY
| PRECEDING
@@ -17690,6 +18151,7 @@ col_name_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| LEAST
| NATIONAL
@@ -18058,6 +18520,7 @@ bare_label_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| KEEP
| KEY
@@ -18097,6 +18560,7 @@ bare_label_keyword:
| NATIONAL
| NATURAL
| NCHAR
+ | NESTED
| NEW
| NEXT
| NFC
@@ -18141,7 +18605,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
| PLACING
+ | PLAN
| PLANS
| POLICY
| POSITION
diff --git a/src/backend/parser/meson.build b/src/backend/parser/meson.build
index 94517a7f7d..0f0e4ac7ca 100644
--- a/src/backend/parser/meson.build
+++ b/src/backend/parser/meson.build
@@ -10,6 +10,7 @@ backend_sources += files(
'parse_enr.c',
'parse_expr.c',
'parse_func.c',
+ 'parse_jsontable.c',
'parse_merge.c',
'parse_node.c',
'parse_oper.c',
diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c
index 334b9b42bd..affd812619 100644
--- a/src/backend/parser/parse_clause.c
+++ b/src/backend/parser/parse_clause.c
@@ -697,7 +697,11 @@ transformRangeTableFunc(ParseState *pstate, RangeTableFunc *rtf)
char **names;
int colno;
- /* Currently only XMLTABLE is supported */
+ /*
+ * Currently we only support XMLTABLE here. See transformJsonTable() for
+ * JSON_TABLE support.
+ */
+ tf->functype = TFT_XMLTABLE;
constructName = "XMLTABLE";
docType = XMLOID;
@@ -1104,13 +1108,17 @@ transformFromClauseItem(ParseState *pstate, Node *n,
rtr->rtindex = nsitem->p_rtindex;
return (Node *) rtr;
}
- else if (IsA(n, RangeTableFunc))
+ else if (IsA(n, RangeTableFunc) || IsA(n, JsonTable))
{
/* table function is like a plain relation */
RangeTblRef *rtr;
ParseNamespaceItem *nsitem;
- nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ if (IsA(n, RangeTableFunc))
+ nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ else
+ nsitem = transformJsonTable(pstate, (JsonTable *) n);
+
*top_nsitem = nsitem;
*namespace = list_make1(nsitem);
rtr = makeNode(RangeTblRef);
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 21979fd64f..de05fa5e70 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4342,6 +4342,19 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
JSON_BEHAVIOR_NULL,
jsexpr->returning);
break;
+
+ case JSON_TABLE_OP:
+ func_name = "JSON_TABLE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->returning = makeNode(JsonReturning);
+ jsexpr->returning->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ jsexpr->returning->typid = exprType(jsexpr->formatted_expr);
+ jsexpr->returning->typmod = -1;
+ jsexpr->on_error = transformJsonBehavior(pstate, func->on_error,
+ JSON_BEHAVIOR_EMPTY,
+ jsexpr->returning);
+ break;
}
Assert(jsexpr != NULL && jsexpr->formatted_expr != NULL);
diff --git a/src/backend/parser/parse_jsontable.c b/src/backend/parser/parse_jsontable.c
new file mode 100644
index 0000000000..05f074a1b2
--- /dev/null
+++ b/src/backend/parser/parse_jsontable.c
@@ -0,0 +1,751 @@
+/*-------------------------------------------------------------------------
+ *
+ * parse_jsontable.c
+ * parsing of JSON_TABLE
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/parser/parse_jsontable.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "catalog/pg_collation.h"
+#include "catalog/pg_type.h"
+#include "miscadmin.h"
+#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
+#include "optimizer/optimizer.h"
+#include "parser/parse_clause.h"
+#include "parser/parse_collate.h"
+#include "parser/parse_expr.h"
+#include "parser/parse_relation.h"
+#include "parser/parse_type.h"
+#include "utils/builtins.h"
+#include "utils/json.h"
+#include "utils/lsyscache.h"
+
+/* Context for JSON_TABLE transformation */
+typedef struct JsonTableParseContext
+{
+ ParseState *pstate; /* parsing state */
+ JsonTable *table; /* untransformed node */
+ TableFunc *tablefunc; /* transformed node */
+ List *pathNames; /* list of all path and columns names */
+ int pathNameId; /* path name id counter */
+ Oid contextItemTypid; /* type oid of context item (json/jsonb) */
+} JsonTableParseContext;
+
+static JsonTableParent *transformJsonTableColumns(JsonTableParseContext * cxt,
+ JsonTablePlan *plan,
+ List *columns,
+ char *pathSpec,
+ char **pathName,
+ int location);
+
+static Node *
+makeStringConst(char *str, int location)
+{
+ A_Const *n = makeNode(A_Const);
+
+ n->val.node.type = T_String;
+ n->val.sval.sval = str;
+ n->location = location;
+
+ return (Node *) n;
+}
+
+/*
+ * Transform JSON_TABLE column
+ * - regular column into JSON_VALUE()
+ * - FORMAT JSON column into JSON_QUERY()
+ * - EXISTS column into JSON_EXISTS()
+ */
+static Node *
+transformJsonTableColumn(JsonTableColumn *jtc, Node *contextItemExpr,
+ List *passingArgs, bool errorOnError)
+{
+ JsonFuncExpr *jfexpr = makeNode(JsonFuncExpr);
+ JsonCommon *common = makeNode(JsonCommon);
+ JsonOutput *output = makeNode(JsonOutput);
+ char *pathspec;
+ JsonFormat *default_format;
+
+ jfexpr->op =
+ jtc->coltype == JTC_REGULAR ? JSON_VALUE_OP :
+ jtc->coltype == JTC_EXISTS ? JSON_EXISTS_OP : JSON_QUERY_OP;
+ jfexpr->common = common;
+ jfexpr->output = output;
+ jfexpr->on_empty = jtc->on_empty;
+ jfexpr->on_error = jtc->on_error;
+ if (!jfexpr->on_error && errorOnError)
+ jfexpr->on_error = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, -1);
+ jfexpr->quotes = jtc->quotes;
+ jfexpr->wrapper = jtc->wrapper;
+ jfexpr->location = jtc->location;
+
+ output->typeName = jtc->typeName;
+ output->returning = makeNode(JsonReturning);
+ output->returning->format = jtc->format;
+
+ default_format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+
+ common->pathname = NULL;
+ /* formatted_expr will be set later in transformJsonValueExpr(). */
+ common->expr = makeJsonValueExpr((Expr *) contextItemExpr, NULL,
+ default_format);
+ common->passing = passingArgs;
+
+ if (jtc->pathspec)
+ pathspec = jtc->pathspec;
+ else
+ {
+ /* Construct default path as '$."column_name"' */
+ StringInfoData path;
+
+ initStringInfo(&path);
+
+ appendStringInfoString(&path, "$.");
+ escape_json(&path, jtc->name);
+
+ pathspec = path.data;
+ }
+
+ common->pathspec = makeStringConst(pathspec, -1);
+
+ return (Node *) jfexpr;
+}
+
+static bool
+isJsonTablePathNameDuplicate(JsonTableParseContext * cxt, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, cxt->pathNames)
+ {
+ if (!strcmp(pathname, (const char *) lfirst(lc)))
+ return true;
+ }
+
+ return false;
+}
+
+/* Register the column name in the path name list. */
+static void
+registerJsonTableColumn(JsonTableParseContext * cxt, char *colname)
+{
+ if (isJsonTablePathNameDuplicate(cxt, colname))
+ ereport(ERROR,
+ (errcode(ERRCODE_DUPLICATE_ALIAS),
+ errmsg("duplicate JSON_TABLE column name: %s", colname),
+ errhint("JSON_TABLE column names must be distinct from one another.")));
+
+ cxt->pathNames = lappend(cxt->pathNames, colname);
+}
+
+/* Recursively register all nested column names in the path name list. */
+static void
+registerAllJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ if (jtc->pathname)
+ registerJsonTableColumn(cxt, jtc->pathname);
+
+ registerAllJsonTableColumns(cxt, jtc->columns);
+ }
+ else
+ {
+ registerJsonTableColumn(cxt, jtc->name);
+ }
+ }
+}
+
+/* Generate a new unique JSON_TABLE path name. */
+static char *
+generateJsonTablePathName(JsonTableParseContext * cxt)
+{
+ char namebuf[32];
+ char *name = namebuf;
+
+ do
+ {
+ snprintf(namebuf, sizeof(namebuf), "json_table_path_%d",
+ ++cxt->pathNameId);
+ } while (isJsonTablePathNameDuplicate(cxt, name));
+
+ name = pstrdup(name);
+ cxt->pathNames = lappend(cxt->pathNames, name);
+
+ return name;
+}
+
+/* Collect sibling path names from plan to the specified list. */
+static void
+collectSiblingPathsInJsonTablePlan(JsonTablePlan *plan, List **paths)
+{
+ if (plan->plan_type == JSTP_SIMPLE)
+ *paths = lappend(*paths, plan->pathname);
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ *paths = lappend(*paths, plan->plan1->pathname);
+ }
+ else if (plan->join_type == JSTPJ_CROSS ||
+ plan->join_type == JSTPJ_UNION)
+ {
+ collectSiblingPathsInJsonTablePlan(plan->plan1, paths);
+ collectSiblingPathsInJsonTablePlan(plan->plan2, paths);
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE join type %d",
+ plan->join_type);
+ }
+}
+
+/*
+ * Validate child JSON_TABLE plan by checking that:
+ * - all nested columns have path names specified
+ * - all nested columns have corresponding node in the sibling plan
+ * - plan does not contain duplicate or extra nodes
+ */
+static void
+validateJsonTableChildPlan(ParseState *pstate, JsonTablePlan *plan,
+ List *columns)
+{
+ ListCell *lc1;
+ List *siblings = NIL;
+ int nchildren = 0;
+
+ if (plan)
+ collectSiblingPathsInJsonTablePlan(plan, &siblings);
+
+ foreach(lc1, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc1));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ ListCell *lc2;
+ bool found = false;
+
+ if (!jtc->pathname)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("nested JSON_TABLE columns must contain an explicit AS pathname specification if an explicit PLAN clause is used"),
+ parser_errposition(pstate, jtc->location)));
+
+ /* find nested path name in the list of sibling path names */
+ foreach(lc2, siblings)
+ {
+ if ((found = !strcmp(jtc->pathname, lfirst(lc2))))
+ break;
+ }
+
+ if (!found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node for nested path %s was not found in plan.", jtc->pathname),
+ parser_errposition(pstate, jtc->location)));
+
+ nchildren++;
+ }
+ }
+
+ if (list_length(siblings) > nchildren)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node contains some extra or duplicate sibling nodes."),
+ parser_errposition(pstate, plan ? plan->location : -1)));
+}
+
+static JsonTableColumn *
+findNestedJsonTableColumn(List *columns, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED &&
+ jtc->pathname &&
+ !strcmp(jtc->pathname, pathname))
+ return jtc;
+ }
+
+ return NULL;
+}
+
+static Node *
+transformNestedJsonTableColumn(JsonTableParseContext * cxt, JsonTableColumn *jtc,
+ JsonTablePlan *plan)
+{
+ JsonTableParent *node;
+ char *pathname = jtc->pathname;
+
+ node = transformJsonTableColumns(cxt, plan, jtc->columns, jtc->pathspec,
+ &pathname, jtc->location);
+
+ return (Node *) node;
+}
+
+static Node *
+makeJsonTableSiblingJoin(bool cross, Node *lnode, Node *rnode)
+{
+ JsonTableSibling *join = makeNode(JsonTableSibling);
+
+ join->larg = lnode;
+ join->rarg = rnode;
+ join->cross = cross;
+
+ return (Node *) join;
+}
+
+/*
+ * Recursively transform child JSON_TABLE plan.
+ *
+ * Default plan is transformed into a cross/union join of its nested columns.
+ * Simple and outer/inner plans are transformed into a JsonTableParent by
+ * finding and transforming corresponding nested column.
+ * Sibling plans are recursively transformed into a JsonTableSibling.
+ */
+static Node *
+transformJsonTableChildPlan(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns)
+{
+ JsonTableColumn *jtc = NULL;
+
+ if (!plan || plan->plan_type == JSTP_DEFAULT)
+ {
+ /* unspecified or default plan */
+ Node *res = NULL;
+ ListCell *lc;
+ bool cross = plan && (plan->join_type & JSTPJ_CROSS);
+
+ /* transform all nested columns into cross/union join */
+ foreach(lc, columns)
+ {
+ JsonTableColumn *col = castNode(JsonTableColumn, lfirst(lc));
+ Node *node;
+
+ if (col->coltype != JTC_NESTED)
+ continue;
+
+ node = transformNestedJsonTableColumn(cxt, col, plan);
+
+ /* join transformed node with previous sibling nodes */
+ res = res ? makeJsonTableSiblingJoin(cross, res, node) : node;
+ }
+
+ return res;
+ }
+ else if (plan->plan_type == JSTP_SIMPLE)
+ {
+ jtc = findNestedJsonTableColumn(columns, plan->pathname);
+ }
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ jtc = findNestedJsonTableColumn(columns, plan->plan1->pathname);
+ }
+ else
+ {
+ Node *node1 = transformJsonTableChildPlan(cxt, plan->plan1,
+ columns);
+ Node *node2 = transformJsonTableChildPlan(cxt, plan->plan2,
+ columns);
+
+ return makeJsonTableSiblingJoin(plan->join_type == JSTPJ_CROSS,
+ node1, node2);
+ }
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE plan type %d", plan->plan_type);
+
+ if (!jtc)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name was %s not found in nested columns list.",
+ plan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ return transformNestedJsonTableColumn(cxt, jtc, plan);
+}
+
+/* Check whether type is json/jsonb, array, or record. */
+static bool
+typeIsComposite(Oid typid)
+{
+ char typtype;
+
+ if (typid == JSONOID ||
+ typid == JSONBOID ||
+ typid == RECORDOID ||
+ type_is_array(typid))
+ return true;
+
+ typtype = get_typtype(typid);
+
+ if (typtype == TYPTYPE_COMPOSITE)
+ return true;
+
+ if (typtype == TYPTYPE_DOMAIN)
+ return typeIsComposite(getBaseType(typid));
+
+ return false;
+}
+
+/* Append transformed non-nested JSON_TABLE columns to the TableFunc node */
+static void
+appendJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *col;
+ ParseState *pstate = cxt->pstate;
+ JsonTable *jt = cxt->table;
+ TableFunc *tf = cxt->tablefunc;
+ bool ordinality_found = false;
+ bool errorOnError = jt->on_error &&
+ jt->on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ foreach(col, columns)
+ {
+ JsonTableColumn *rawc = castNode(JsonTableColumn, lfirst(col));
+ Oid typid;
+ int32 typmod;
+ Node *colexpr;
+
+ if (rawc->name)
+ {
+ /* make sure column names are unique */
+ ListCell *colname;
+
+ foreach(colname, tf->colnames)
+ if (!strcmp((const char *) colname, rawc->name))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("column name \"%s\" is not unique",
+ rawc->name),
+ parser_errposition(pstate, rawc->location)));
+
+ tf->colnames = lappend(tf->colnames,
+ makeString(pstrdup(rawc->name)));
+ }
+
+ /*
+ * Determine the type and typmod for the new column. FOR ORDINALITY
+ * columns are INTEGER by standard; the others are user-specified.
+ */
+ switch (rawc->coltype)
+ {
+ case JTC_FOR_ORDINALITY:
+ if (ordinality_found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use more than one FOR ORDINALITY column"),
+ parser_errposition(pstate, rawc->location)));
+ ordinality_found = true;
+ colexpr = NULL;
+ typid = INT4OID;
+ typmod = -1;
+ break;
+
+ case JTC_REGULAR:
+ typenameTypeIdAndMod(pstate, rawc->typeName, &typid, &typmod);
+
+ /*
+ * Use implicit FORMAT JSON for composite types (arrays and
+ * records).
+ *
+ * For scalar columns, require an explicit FORMAT JSON clause
+ * to get the specified WRAPPER and QUOTES behavior.
+ */
+ if (typeIsComposite(typid))
+ rawc->coltype = JTC_FORMATTED;
+ else if (rawc->wrapper != JSW_NONE)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use WITH WRAPPER clause for scalar columns without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+ else if (rawc->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use OMIT/KEEP QUOTES clause for scalar columns without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+
+ /* FALLTHROUGH */
+ case JTC_EXISTS:
+ case JTC_FORMATTED:
+ {
+ Node *je;
+ CaseTestExpr *param = makeNode(CaseTestExpr);
+
+ param->collation = InvalidOid;
+ param->typeId = cxt->contextItemTypid;
+ param->typeMod = -1;
+
+ je = transformJsonTableColumn(rawc, (Node *) param,
+ NIL, errorOnError);
+
+ colexpr = transformExpr(pstate, je, EXPR_KIND_FROM_FUNCTION);
+ assign_expr_collations(pstate, colexpr);
+
+ typid = exprType(colexpr);
+ typmod = exprTypmod(colexpr);
+ break;
+ }
+
+ case JTC_NESTED:
+ continue;
+
+ default:
+ elog(ERROR, "unknown JSON_TABLE column type: %d", rawc->coltype);
+ break;
+ }
+
+ tf->coltypes = lappend_oid(tf->coltypes, typid);
+ tf->coltypmods = lappend_int(tf->coltypmods, typmod);
+ tf->colcollations = lappend_oid(tf->colcollations, get_typcollation(typid));
+ tf->colvalexprs = lappend(tf->colvalexprs, colexpr);
+ }
+}
+
+/*
+ * Create transformed JSON_TABLE parent plan node by appending all non-nested
+ * columns to the TableFunc node and remembering their indices in the
+ * colvalexprs list.
+ */
+static JsonTableParent *
+makeParentJsonTableNode(JsonTableParseContext * cxt, char *pathSpec, char *pathName,
+ List *columns)
+{
+ JsonTableParent *node = makeNode(JsonTableParent);
+
+ node->path = makeNode(JsonTablePath);
+ node->path->value = makeConst(JSONPATHOID, -1, InvalidOid, -1,
+ DirectFunctionCall1(jsonpath_in,
+ CStringGetDatum(pathSpec)),
+ false, false);
+ if (pathName)
+ node->path->name = pstrdup(pathName);
+
+ /* save start of column range */
+ node->colMin = list_length(cxt->tablefunc->colvalexprs);
+
+ appendJsonTableColumns(cxt, columns);
+
+ /* save end of column range */
+ node->colMax = list_length(cxt->tablefunc->colvalexprs) - 1;
+
+ node->errorOnError =
+ cxt->table->on_error &&
+ cxt->table->on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ return node;
+}
+
+static JsonTableParent *
+transformJsonTableColumns(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns, char *pathSpec, char **pathName,
+ int location)
+{
+ JsonTableParent *node;
+ JsonTablePlan *childPlan;
+ bool defaultPlan = !plan || plan->plan_type == JSTP_DEFAULT;
+
+ if (!*pathName)
+ {
+ if (cxt->table->plan)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE expression"),
+ errdetail("JSON_TABLE columns must contain "
+ "explicit AS pathname specification if "
+ "explicit PLAN clause is used"),
+ parser_errposition(cxt->pstate, location)));
+
+ *pathName = generateJsonTablePathName(cxt);
+ }
+
+ if (defaultPlan)
+ childPlan = plan;
+ else
+ {
+ /* validate parent and child plans */
+ JsonTablePlan *parentPlan;
+
+ if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type != JSTPJ_INNER &&
+ plan->join_type != JSTPJ_OUTER)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Expected INNER or OUTER JSON_TABLE plan node."),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ parentPlan = plan->plan1;
+ childPlan = plan->plan2;
+
+ Assert(parentPlan->plan_type != JSTP_JOINED);
+ Assert(parentPlan->pathname);
+ }
+ else
+ {
+ parentPlan = plan;
+ childPlan = NULL;
+ }
+
+ if (strcmp(parentPlan->pathname, *pathName))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name mismatch: expected %s but %s is given.",
+ *pathName, parentPlan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ validateJsonTableChildPlan(cxt->pstate, childPlan, columns);
+ }
+
+ /* transform only non-nested columns */
+ node = makeParentJsonTableNode(cxt, pathSpec, *pathName, columns);
+
+ if (childPlan || defaultPlan)
+ {
+ /* transform recursively nested columns */
+ node->child = transformJsonTableChildPlan(cxt, childPlan, columns);
+ if (node->child)
+ node->outerJoin = !plan || (plan->join_type & JSTPJ_OUTER);
+ /* else: default plan case, no children found */
+ }
+
+ return node;
+}
+
+/*
+ * transformJsonTable -
+ * Transform a raw JsonTable into TableFunc.
+ *
+ * Transform the document-generating expression, the row-generating expression,
+ * the column-generating expressions, and the default value expressions.
+ */
+ParseNamespaceItem *
+transformJsonTable(ParseState *pstate, JsonTable *jt)
+{
+ JsonTableParseContext cxt;
+ TableFunc *tf = makeNode(TableFunc);
+ JsonFuncExpr *jfe = makeNode(JsonFuncExpr);
+ JsonExpr *je;
+ JsonTablePlan *plan = jt->plan;
+ JsonCommon *jscommon;
+ char *rootPathName = jt->common->pathname;
+ char *rootPath;
+ bool is_lateral;
+
+ cxt.pstate = pstate;
+ cxt.table = jt;
+ cxt.tablefunc = tf;
+ cxt.pathNames = NIL;
+ cxt.pathNameId = 0;
+
+ if (rootPathName)
+ registerJsonTableColumn(&cxt, rootPathName);
+
+ registerAllJsonTableColumns(&cxt, jt->columns);
+
+#if 0 /* XXX it' unclear from the standard whether
+ * root path name is mandatory or not */
+ if (plan && plan->plan_type != JSTP_DEFAULT && !rootPathName)
+ {
+ /* Assign root path name and create corresponding plan node */
+ JsonTablePlan *rootNode = makeNode(JsonTablePlan);
+ JsonTablePlan *rootPlan = (JsonTablePlan *)
+ makeJsonTableJoinedPlan(JSTPJ_OUTER, (Node *) rootNode,
+ (Node *) plan, jt->location);
+
+ rootPathName = generateJsonTablePathName(&cxt);
+
+ rootNode->plan_type = JSTP_SIMPLE;
+ rootNode->pathname = rootPathName;
+
+ plan = rootPlan;
+ }
+#endif
+
+ jscommon = copyObject(jt->common);
+ jscommon->pathspec = makeStringConst(pstrdup("$"), -1);
+
+ jfe->op = JSON_TABLE_OP;
+ jfe->common = jscommon;
+ jfe->on_error = jt->on_error;
+ jfe->location = jt->common->location;
+
+ /*
+ * We make lateral_only names of this level visible, whether or not the
+ * RangeTableFunc is explicitly marked LATERAL. This is needed for SQL
+ * spec compliance and seems useful on convenience grounds for all
+ * functions in FROM.
+ *
+ * (LATERAL can't nest within a single pstate level, so we don't need
+ * save/restore logic here.)
+ */
+ Assert(!pstate->p_lateral_active);
+ pstate->p_lateral_active = true;
+
+ tf->functype = TFT_JSON_TABLE;
+ tf->docexpr = transformExpr(pstate, (Node *) jfe, EXPR_KIND_FROM_FUNCTION);
+
+ cxt.contextItemTypid = exprType(tf->docexpr);
+
+ if (!IsA(jt->common->pathspec, A_Const) ||
+ castNode(A_Const, jt->common->pathspec)->val.node.type != T_String)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("only string constants supported in JSON_TABLE path specification"),
+ parser_errposition(pstate,
+ exprLocation(jt->common->pathspec))));
+
+ rootPath = castNode(A_Const, jt->common->pathspec)->val.sval.sval;
+
+ tf->plan = (Node *) transformJsonTableColumns(&cxt, plan, jt->columns,
+ rootPath, &rootPathName,
+ jt->common->location);
+
+ /* Also save a copy of the PASSING arguments in the TableFunc node. */
+ je = (JsonExpr *) tf->docexpr;
+ tf->passingvalexprs = copyObject(je->passing_values);
+
+ tf->ordinalitycol = -1; /* undefine ordinality column number */
+ tf->location = jt->location;
+
+ pstate->p_lateral_active = false;
+
+ /*
+ * Mark the RTE as LATERAL if the user said LATERAL explicitly, or if
+ * there are any lateral cross-references in it.
+ */
+ is_lateral = jt->lateral || contain_vars_of_level((Node *) tf, 0);
+
+ return addRangeTableEntryForTableFunc(pstate,
+ tf, jt->alias, is_lateral, true);
+}
diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c
index 864ea9b0d5..3ac7e113f0 100644
--- a/src/backend/parser/parse_relation.c
+++ b/src/backend/parser/parse_relation.c
@@ -2073,7 +2073,8 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
Assert(list_length(tf->coltypmods) == list_length(tf->colnames));
Assert(list_length(tf->colcollations) == list_length(tf->colnames));
- refname = alias ? alias->aliasname : pstrdup("xmltable");
+ refname = alias ? alias->aliasname :
+ pstrdup(tf->functype == TFT_XMLTABLE ? "xmltable" : "json_table");
rte->rtekind = RTE_TABLEFUNC;
rte->relid = InvalidOid;
@@ -2096,7 +2097,7 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
errmsg("%s function has %d columns available but %d columns specified",
- "XMLTABLE",
+ tf->functype == TFT_XMLTABLE ? "XMLTABLE" : "JSON_TABLE",
list_length(tf->colnames), numaliases)));
rte->eref = eref;
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 8f81624f97..ca7228ea41 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -2002,6 +2002,9 @@ FigureColnameInternal(Node *node, char **name)
case JSON_VALUE_OP:
*name = "json_value";
return 2;
+ case JSON_TABLE_OP:
+ *name = "json_table";
+ return 2;
}
break;
default:
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 6c5602c64d..79632e3dfd 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -61,10 +61,12 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
+#include "executor/execExpr.h"
#include "funcapi.h"
#include "lib/stringinfo.h"
#include "miscadmin.h"
#include "nodes/miscnodes.h"
+#include "nodes/nodeFuncs.h"
#include "regex/regex.h"
#include "utils/builtins.h"
#include "utils/date.h"
@@ -75,6 +77,8 @@
#include "utils/guc.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
#include "utils/timestamp.h"
#include "utils/varlena.h"
@@ -156,6 +160,61 @@ typedef struct JsonValueListIterator
ListCell *next;
} JsonValueListIterator;
+/* Structures for JSON_TABLE execution */
+
+typedef enum JsonTablePlanStateType
+{
+ JSON_TABLE_SCAN_STATE = 0,
+ JSON_TABLE_JOIN_STATE
+} JsonTablePlanStateType;
+
+typedef struct JsonTablePlanState
+{
+ JsonTablePlanStateType type;
+
+ struct JsonTablePlanState *parent;
+ struct JsonTablePlanState *nested;
+} JsonTablePlanState;
+
+typedef struct JsonTableScanState
+{
+ JsonTablePlanState plan;
+
+ MemoryContext mcxt;
+ JsonPath *path;
+ List *args;
+ JsonValueList found;
+ JsonValueListIterator iter;
+ Datum current;
+ int ordinal;
+ bool currentIsNull;
+ bool outerJoin;
+ bool errorOnError;
+ bool advanceNested;
+ bool reset;
+} JsonTableScanState;
+
+typedef struct JsonTableJoinState
+{
+ JsonTablePlanState plan;
+
+ JsonTablePlanState *left;
+ JsonTablePlanState *right;
+ bool cross;
+ bool advanceRight;
+} JsonTableJoinState;
+
+/* random number to identify JsonTableExecContext */
+#define JSON_TABLE_EXEC_CONTEXT_MAGIC 418352867
+
+typedef struct JsonTableExecContext
+{
+ int magic;
+ JsonTableScanState **colexprscans;
+ JsonTableScanState *root;
+ bool empty;
+} JsonTableExecContext;
+
/* strict/lax flags is decomposed into four [un]wrap/error flags */
#define jspStrictAbsenseOfErrors(cxt) (!(cxt)->laxMode)
#define jspAutoUnwrap(cxt) ((cxt)->laxMode)
@@ -248,6 +307,7 @@ static JsonPathExecResult getArrayIndex(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonbValue *jb, int32 *index);
static JsonBaseObjectInfo setBaseObject(JsonPathExecContext *cxt,
JsonbValue *jbv, int32 id);
+static void JsonValueListClear(JsonValueList *jvl);
static void JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv);
static int JsonValueListLength(const JsonValueList *jvl);
static bool JsonValueListIsEmpty(JsonValueList *jvl);
@@ -265,6 +325,14 @@ static JsonbValue *wrapItemsInArray(const JsonValueList *items);
static int compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
bool useTz, bool *cast_error);
+
+static JsonTablePlanState * JsonTableInitPlanState(JsonTableExecContext * cxt,
+ Node *plan,
+ JsonTablePlanState * parent);
+static bool JsonTablePlanNextRow(JsonTablePlanState * state);
+static bool JsonTableScanNextRow(JsonTableScanState *scan);
+
+
/****************** User interface to JsonPath executor ********************/
/*
@@ -2518,6 +2586,13 @@ setBaseObject(JsonPathExecContext *cxt, JsonbValue *jbv, int32 id)
return baseObject;
}
+static void
+JsonValueListClear(JsonValueList *jvl)
+{
+ jvl->singleton = NULL;
+ jvl->list = NULL;
+}
+
static void
JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv)
{
@@ -3123,3 +3198,471 @@ JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
}
}
+
+/************************ JSON_TABLE functions ***************************/
+
+/*
+ * Returns private data from executor state. Ensure validity by check with
+ * MAGIC number.
+ */
+static inline JsonTableExecContext *
+GetJsonTableExecContext(TableFuncScanState *state, const char *fname)
+{
+ JsonTableExecContext *result;
+
+ if (!IsA(state, TableFuncScanState))
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+ result = (JsonTableExecContext *) state->opaque;
+ if (result->magic != JSON_TABLE_EXEC_CONTEXT_MAGIC)
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+
+ return result;
+}
+
+/* Recursively initialize JSON_TABLE scan / join state */
+static JsonTableJoinState *
+JsonTableInitJoinState(JsonTableExecContext * cxt, JsonTableSibling *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTableJoinState *join = palloc0(sizeof(*join));
+
+ join->plan.type = JSON_TABLE_JOIN_STATE;
+ /* parent and nested not set. */
+
+ join->cross = plan->cross;
+ join->left = JsonTableInitPlanState(cxt, plan->larg, parent);
+ join->right = JsonTableInitPlanState(cxt, plan->rarg, parent);
+
+ return join;
+}
+
+static JsonTableScanState *
+JsonTableInitScanState(JsonTableExecContext * cxt, JsonTableParent *plan,
+ JsonTablePlanState * parent,
+ List *args, MemoryContext mcxt)
+{
+ JsonTableScanState *scan = palloc0(sizeof(*scan));
+ int i;
+
+ scan->plan.type = JSON_TABLE_SCAN_STATE;
+ scan->plan.parent = parent;
+
+ scan->outerJoin = plan->outerJoin;
+ scan->errorOnError = plan->errorOnError;
+ scan->path = DatumGetJsonPathP(plan->path->value->constvalue);
+ scan->args = args;
+ scan->mcxt = AllocSetContextCreate(mcxt, "JsonTableExecContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Set after settings scan->args and scan->mcxt, because the recursive
+ * call wants to use those values.
+ */
+ scan->plan.nested = plan->child ?
+ JsonTableInitPlanState(cxt, plan->child, (JsonTablePlanState *) scan) :
+ NULL;
+
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+
+ for (i = plan->colMin; i <= plan->colMax; i++)
+ cxt->colexprscans[i] = scan;
+
+ return scan;
+}
+
+/* Recursively initialize JSON_TABLE scan state */
+static JsonTablePlanState *
+JsonTableInitPlanState(JsonTableExecContext * cxt, Node *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTablePlanState *state;
+
+ if (IsA(plan, JsonTableSibling))
+ {
+ JsonTableSibling *join = castNode(JsonTableSibling, plan);
+
+ state = (JsonTablePlanState *)
+ JsonTableInitJoinState(cxt, join, parent);
+ }
+ else
+ {
+ JsonTableParent *scan = castNode(JsonTableParent, plan);
+ JsonTableScanState *parent_scan = (JsonTableScanState *) parent;
+
+ Assert(parent_scan);
+ state = (JsonTablePlanState *)
+ JsonTableInitScanState(cxt, scan, parent, parent_scan->args,
+ parent_scan->mcxt);
+ }
+
+ return state;
+}
+
+/*
+ * JsonTableInitOpaque
+ * Fill in TableFuncScanState->opaque for JsonTable processor
+ */
+static void
+JsonTableInitOpaque(TableFuncScanState *state, int natts)
+{
+ JsonTableExecContext *cxt;
+ PlanState *ps = &state->ss.ps;
+ TableFuncScan *tfs = castNode(TableFuncScan, ps->plan);
+ TableFunc *tf = tfs->tablefunc;
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+ JsonExpr *je = castNode(JsonExpr, tf->docexpr);
+ List *args = NIL;
+
+ cxt = palloc0(sizeof(JsonTableExecContext));
+ cxt->magic = JSON_TABLE_EXEC_CONTEXT_MAGIC;
+
+ if (state->passingvalexprs)
+ {
+ ListCell *exprlc;
+ ListCell *namelc;
+
+ Assert(list_length(state->passingvalexprs) ==
+ list_length(je->passing_names));
+ forboth(exprlc, state->passingvalexprs,
+ namelc, je->passing_names)
+ {
+ ExprState *state = lfirst_node(ExprState, exprlc);
+ String *name = lfirst_node(String, namelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = pstrdup(name->sval);
+ var->typid = exprType((Node *) state->expr);
+ var->typmod = exprTypmod((Node *) state->expr);
+
+ /*
+ * Evaluate the expression and save the value to be returned by
+ * GetJsonPathVar().
+ */
+ var->value = ExecEvalExpr(state, ps->ps_ExprContext,
+ &var->isnull);
+
+ args = lappend(args, var);
+ }
+ }
+
+ cxt->colexprscans = palloc(sizeof(JsonTableScanState *) *
+ list_length(tf->colvalexprs));
+
+ cxt->root = JsonTableInitScanState(cxt, root, NULL, args,
+ CurrentMemoryContext);
+ state->opaque = cxt;
+}
+
+/* Reset scan iterator to the beginning of the item list */
+static void
+JsonTableRescan(JsonTableScanState *scan)
+{
+ JsonValueListInitIterator(&scan->found, &scan->iter);
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ scan->advanceNested = false;
+ scan->ordinal = 0;
+}
+
+/* Reset context item of a scan, execute JSON path and reset a scan */
+static void
+JsonTableResetContextItem(JsonTableScanState *scan, Datum item)
+{
+ MemoryContext oldcxt;
+ JsonPathExecResult res;
+ Jsonb *js = (Jsonb *) DatumGetJsonbP(item);
+
+ JsonValueListClear(&scan->found);
+
+ MemoryContextResetOnly(scan->mcxt);
+
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+
+ res = executeJsonPath(scan->path, scan->args, GetJsonPathVar, js,
+ scan->errorOnError, &scan->found,
+ false /* FIXME */ );
+
+ MemoryContextSwitchTo(oldcxt);
+
+ if (jperIsError(res))
+ {
+ Assert(!scan->errorOnError);
+ JsonValueListClear(&scan->found); /* EMPTY ON ERROR case */
+ }
+
+ JsonTableRescan(scan);
+}
+
+/*
+ * JsonTableSetDocument
+ * Install the input document
+ */
+static void
+JsonTableSetDocument(TableFuncScanState *state, Datum value)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableSetDocument");
+
+ JsonTableResetContextItem(cxt->root, value);
+}
+
+/* Recursively reset scan and its child nodes */
+static void
+JsonTableRescanRecursive(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTableRescanRecursive(join->left);
+ JsonTableRescanRecursive(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan = (JsonTableScanState *) state;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ JsonTableRescan(scan);
+ if (scan->plan.nested)
+ JsonTableRescanRecursive(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a cross/union joined scan.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTablePlanNextRow(JsonTablePlanState * state)
+{
+ JsonTableJoinState *join;
+
+ if (state->type == JSON_TABLE_SCAN_STATE)
+ return JsonTableScanNextRow((JsonTableScanState *) state);
+
+ join = (JsonTableJoinState *) state;
+ if (join->advanceRight)
+ {
+ /* fetch next inner row */
+ if (JsonTablePlanNextRow(join->right))
+ return true;
+
+ /* inner rows are exhausted */
+ if (join->cross)
+ join->advanceRight = false; /* next outer row */
+ else
+ return false; /* end of scan */
+ }
+
+ while (!join->advanceRight)
+ {
+ /* fetch next outer row */
+ bool left = JsonTablePlanNextRow(join->left);
+
+ if (join->cross)
+ {
+ if (!left)
+ return false; /* end of scan */
+
+ JsonTableRescanRecursive(join->right);
+
+ if (!JsonTablePlanNextRow(join->right))
+ continue; /* next outer row */
+
+ join->advanceRight = true; /* next inner row */
+ }
+ else if (!left)
+ {
+ if (!JsonTablePlanNextRow(join->right))
+ return false; /* end of scan */
+
+ join->advanceRight = true; /* next inner row */
+ }
+
+ break;
+ }
+
+ return true;
+}
+
+/* Recursively set 'reset' flag of scan and its child nodes */
+static void
+JsonTablePlanReset(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTablePlanReset(join->left);
+ JsonTablePlanReset(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ scan = (JsonTableScanState *) state;
+ scan->reset = true;
+ scan->advanceNested = false;
+
+ if (scan->plan.nested)
+ JsonTablePlanReset(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a simple scan with outer/inner joined nested subscans.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTableScanNextRow(JsonTableScanState *scan)
+{
+ /* reset context item if requested */
+ if (scan->reset)
+ {
+ JsonTableScanState *parent_scan =
+ (JsonTableScanState *) scan->plan.parent;
+
+ Assert(parent_scan && !parent_scan->currentIsNull);
+ JsonTableResetContextItem(scan, parent_scan->current);
+ scan->reset = false;
+ }
+
+ if (scan->advanceNested)
+ {
+ /* fetch next nested row */
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested)
+ return true;
+ }
+
+ for (;;)
+ {
+ /* fetch next row */
+ JsonbValue *jbv = JsonValueListNext(&scan->found, &scan->iter);
+ MemoryContext oldcxt;
+
+ if (!jbv)
+ {
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ return false; /* end of scan */
+ }
+
+ /* set current row item */
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+ scan->current = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ scan->currentIsNull = false;
+ MemoryContextSwitchTo(oldcxt);
+
+ scan->ordinal++;
+
+ if (!scan->plan.nested)
+ break;
+
+ JsonTablePlanReset(scan->plan.nested);
+
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested || scan->outerJoin)
+ break;
+ }
+
+ return true;
+}
+
+/*
+ * JsonTableFetchRow
+ * Prepare the next "current" tuple for upcoming GetValue calls.
+ * Returns FALSE if the row-filter expression returned no more rows.
+ */
+static bool
+JsonTableFetchRow(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableFetchRow");
+
+ if (cxt->empty)
+ return false;
+
+ return JsonTableScanNextRow(cxt->root);
+}
+
+/*
+ * JsonTableGetValue
+ * Return the value for column number 'colnum' for the current row.
+ *
+ * This leaks memory, so be sure to reset often the context in which it's
+ * called.
+ */
+static Datum
+JsonTableGetValue(TableFuncScanState *state, int colnum,
+ Oid typid, int32 typmod, bool *isnull)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableGetValue");
+ ExprContext *econtext = state->ss.ps.ps_ExprContext;
+ ExprState *estate = list_nth(state->colvalexprs, colnum);
+ JsonTableScanState *scan = cxt->colexprscans[colnum];
+ Datum result;
+
+ if (scan->currentIsNull) /* NULL from outer/union join */
+ {
+ result = (Datum) 0;
+ *isnull = true;
+ }
+ else if (estate) /* regular column */
+ {
+ Datum saved_caseValue = econtext->caseValue_datum;
+ bool saved_caseIsNull = econtext->caseValue_isNull;
+
+ /* Pass the value for CaseTestExpr that may be present in colexpr */
+ econtext->caseValue_datum = scan->current;
+ econtext->caseValue_isNull = false;
+
+ result = ExecEvalExpr(estate, econtext, isnull);
+
+ econtext->caseValue_datum = saved_caseValue;
+ econtext->caseValue_isNull = saved_caseIsNull;
+ }
+ else
+ {
+ result = Int32GetDatum(scan->ordinal); /* ordinality column */
+ *isnull = false;
+ }
+
+ return result;
+}
+
+/*
+ * JsonTableDestroyOpaque
+ */
+static void
+JsonTableDestroyOpaque(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableDestroyOpaque");
+
+ /* not valid anymore */
+ cxt->magic = 0;
+
+ state->opaque = NULL;
+}
+
+const TableFuncRoutine JsonbTableRoutine =
+{
+ JsonTableInitOpaque,
+ JsonTableSetDocument,
+ NULL,
+ NULL,
+ NULL,
+ JsonTableFetchRow,
+ JsonTableGetValue,
+ JsonTableDestroyOpaque
+};
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index d20b96780c..27598c2d37 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -522,6 +522,8 @@ static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
static void get_json_path_spec(Node *path_spec, deparse_context *context,
bool showimplicit);
+static void get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8627,7 +8629,8 @@ get_json_behavior(JsonBehavior *behavior, deparse_context *context,
/*
* get_json_expr_options
*
- * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS and
+ * JSON_TABLE columns.
*/
static void
get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
@@ -9875,6 +9878,9 @@ get_rule_expr(Node *node, deparse_context *context,
case JSON_VALUE_OP:
appendStringInfoString(buf, "JSON_VALUE(");
break;
+ default:
+ elog(ERROR, "unexpected JsonExpr type: %d", jexpr->op);
+ break;
}
get_rule_expr(jexpr->formatted_expr, context, showimplicit);
@@ -11231,16 +11237,14 @@ get_sublink_expr(SubLink *sublink, deparse_context *context)
/* ----------
- * get_tablefunc - Parse back a table function
+ * get_xmltable - Parse back a XMLTABLE function
* ----------
*/
static void
-get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+get_xmltable(TableFunc *tf, deparse_context *context, bool showimplicit)
{
StringInfo buf = context->buf;
- /* XMLTABLE is the only existing implementation. */
-
appendStringInfoString(buf, "XMLTABLE(");
if (tf->ns_uris != NIL)
@@ -11331,6 +11335,271 @@ get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
appendStringInfoChar(buf, ')');
}
+/*
+ * get_json_nested_columns - Parse back nested JSON_TABLE columns
+ */
+static void
+get_json_table_nested_columns(TableFunc *tf, Node *node,
+ deparse_context *context, bool showimplicit,
+ bool needcomma)
+{
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_nested_columns(tf, n->larg, context, showimplicit,
+ needcomma);
+ get_json_table_nested_columns(tf, n->rarg, context, showimplicit, true);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ if (needcomma)
+ appendStringInfoChar(context->buf, ',');
+
+ appendStringInfoChar(context->buf, ' ');
+ appendContextKeyword(context, "NESTED PATH ", 0, 0, 0);
+ get_const_expr(n->path->value, context, -1);
+ appendStringInfo(context->buf, " AS %s", quote_identifier(n->path->name));
+ get_json_table_columns(tf, n, context, showimplicit);
+ }
+}
+
+/*
+ * get_json_table_plan - Parse back a JSON_TABLE plan
+ */
+static void
+get_json_table_plan(TableFunc *tf, Node *node, deparse_context *context,
+ bool parenthesize)
+{
+ if (parenthesize)
+ appendStringInfoChar(context->buf, '(');
+
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_plan(tf, n->larg, context,
+ IsA(n->larg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->larg)->child);
+
+ appendStringInfoString(context->buf, n->cross ? " CROSS " : " UNION ");
+
+ get_json_table_plan(tf, n->rarg, context,
+ IsA(n->rarg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->rarg)->child);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ appendStringInfoString(context->buf, quote_identifier(n->path->name));
+
+ if (n->child)
+ {
+ appendStringInfoString(context->buf,
+ n->outerJoin ? " OUTER " : " INNER ");
+ get_json_table_plan(tf, n->child, context,
+ IsA(n->child, JsonTableSibling));
+ }
+ }
+
+ if (parenthesize)
+ appendStringInfoChar(context->buf, ')');
+}
+
+/*
+ * get_json_table_columns - Parse back JSON_TABLE columns
+ */
+static void
+get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ ListCell *lc_colname;
+ ListCell *lc_coltype;
+ ListCell *lc_coltypmod;
+ ListCell *lc_colvalexpr;
+ int colnum = 0;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "COLUMNS (", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forfour(lc_colname, tf->colnames,
+ lc_coltype, tf->coltypes,
+ lc_coltypmod, tf->coltypmods,
+ lc_colvalexpr, tf->colvalexprs)
+ {
+ char *colname = strVal(lfirst(lc_colname));
+ JsonExpr *colexpr;
+ Oid typid;
+ int32 typmod;
+ bool ordinality;
+ JsonBehaviorType default_behavior;
+
+ typid = lfirst_oid(lc_coltype);
+ typmod = lfirst_int(lc_coltypmod);
+ colexpr = castNode(JsonExpr, lfirst(lc_colvalexpr));
+
+ if (colnum < node->colMin)
+ {
+ colnum++;
+ continue;
+ }
+
+ if (colnum > node->colMax)
+ break;
+
+ if (colnum > node->colMin)
+ appendStringInfoString(buf, ", ");
+
+ colnum++;
+
+ ordinality = !colexpr;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ appendStringInfo(buf, "%s %s", quote_identifier(colname),
+ ordinality ? "FOR ORDINALITY" :
+ format_type_with_typemod(typid, typmod));
+ if (ordinality)
+ continue;
+
+ if (colexpr->op == JSON_EXISTS_OP)
+ {
+ appendStringInfoString(buf, " EXISTS");
+ default_behavior = JSON_BEHAVIOR_FALSE;
+ }
+ else
+ {
+ if (colexpr->op == JSON_QUERY_OP)
+ {
+ char typcategory;
+ bool typispreferred;
+
+ get_type_category_preferred(typid, &typcategory, &typispreferred);
+
+ if (typcategory == TYPCATEGORY_STRING)
+ appendStringInfoString(buf,
+ colexpr->format->format_type == JS_FORMAT_JSONB ?
+ " FORMAT JSONB" : " FORMAT JSON");
+ }
+
+ default_behavior = JSON_BEHAVIOR_NULL;
+ }
+
+ if (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR)
+ default_behavior = JSON_BEHAVIOR_ERROR;
+
+ appendStringInfoString(buf, " PATH ");
+
+ get_json_path_spec(colexpr->path_spec, context, showimplicit);
+
+ get_json_expr_options(colexpr, context, default_behavior);
+ }
+
+ if (node->child)
+ get_json_table_nested_columns(tf, node->child, context, showimplicit,
+ node->colMax >= node->colMin);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_json_table - Parse back a JSON_TABLE function
+ * ----------
+ */
+static void
+get_json_table(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+
+ appendStringInfoString(buf, "JSON_TABLE(");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_const_expr(root->path->value, context, -1);
+
+ appendStringInfo(buf, " AS %s", quote_identifier(root->path->name));
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PASSING ", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr((Node *) lfirst(lc2), context, false);
+ appendStringInfo(buf, " AS %s",
+ quote_identifier((lfirst_node(String, lc1))->sval)
+ );
+ }
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+ }
+
+ get_json_table_columns(tf, root, context, showimplicit);
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PLAN ", 0, 0, 0);
+ get_json_table_plan(tf, (Node *) root, context, true);
+
+ if (jexpr->on_error->btype != JSON_BEHAVIOR_EMPTY)
+ get_json_behavior(jexpr->on_error, context, "ERROR");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_tablefunc - Parse back a table function
+ * ----------
+ */
+static void
+get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ /* XMLTABLE and JSON_TABLE are the only existing implementations. */
+
+ if (tf->functype == TFT_XMLTABLE)
+ get_xmltable(tf, context, showimplicit);
+ else if (tf->functype == TFT_JSON_TABLE)
+ get_json_table(tf, context, showimplicit);
+}
+
/* ----------
* get_from_clause - Parse back a FROM clause
*
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 61d97b30bc..fddbb8575a 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1883,6 +1883,8 @@ typedef struct TableFuncScanState
ExprState *rowexpr; /* state for row-generating expression */
List *colexprs; /* state for column-generating expression */
List *coldefexprs; /* state for column default expressions */
+ List *colvalexprs; /* state for column value expression */
+ List *passingvalexprs; /* state for PASSING argument expression */
List *ns_names; /* same as TableFunc.ns_names */
List *ns_uris; /* list of states of namespace URI exprs */
Bitmapset *notnulls; /* nullability flag for each output column */
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index a850a1928b..a0b864deda 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -113,6 +113,8 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
+extern Node *makeJsonTableJoinedPlan(JsonTablePlanJoinType type,
+ Node *plan1, Node *plan2, int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index c1f9ef844e..652b61a1d3 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1733,6 +1733,19 @@ typedef enum JsonQuotes
*/
typedef char *JsonPathSpec;
+/*
+ * JsonTableColumnType -
+ * enumeration of JSON_TABLE column types
+ */
+typedef enum JsonTableColumnType
+{
+ JTC_FOR_ORDINALITY,
+ JTC_REGULAR,
+ JTC_EXISTS,
+ JTC_FORMATTED,
+ JTC_NESTED,
+} JsonTableColumnType;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1786,6 +1799,83 @@ typedef struct JsonFuncExpr
int location; /* token location, or -1 if unknown */
} JsonFuncExpr;
+/*
+ * JsonTableColumn -
+ * untransformed representation of JSON_TABLE column
+ */
+typedef struct JsonTableColumn
+{
+ NodeTag type;
+ JsonTableColumnType coltype; /* column type */
+ char *name; /* column name */
+ TypeName *typeName; /* column type name */
+ char *pathspec; /* path specification, if any */
+ char *pathname; /* path name, if any */
+ JsonFormat *format; /* JSON format clause, if specified */
+ JsonWrapper wrapper; /* WRAPPER behavior for formatted columns */
+ JsonQuotes quotes; /* omit or keep quotes on scalar strings? */
+ List *columns; /* nested columns */
+ JsonBehavior *on_empty; /* ON EMPTY behavior */
+ JsonBehavior *on_error; /* ON ERROR behavior */
+ int location; /* token location, or -1 if unknown */
+} JsonTableColumn;
+
+/*
+ * JsonTablePlanType -
+ * flags for JSON_TABLE plan node types representation
+ */
+typedef enum JsonTablePlanType
+{
+ JSTP_DEFAULT,
+ JSTP_SIMPLE,
+ JSTP_JOINED,
+} JsonTablePlanType;
+
+/*
+ * JsonTablePlanJoinType -
+ * flags for JSON_TABLE join types representation
+ */
+typedef enum JsonTablePlanJoinType
+{
+ JSTPJ_INNER = 0x01,
+ JSTPJ_OUTER = 0x02,
+ JSTPJ_CROSS = 0x04,
+ JSTPJ_UNION = 0x08,
+} JsonTablePlanJoinType;
+
+typedef struct JsonTablePlan JsonTablePlan;
+
+/*
+ * JsonTablePlan -
+ * untransformed representation of JSON_TABLE plan node
+ */
+struct JsonTablePlan
+{
+ NodeTag type;
+ JsonTablePlanType plan_type; /* plan type */
+ JsonTablePlanJoinType join_type; /* join type (for joined plan only) */
+ JsonTablePlan *plan1; /* first joined plan */
+ JsonTablePlan *plan2; /* second joined plan */
+ char *pathname; /* path name (for simple plan only) */
+ int location; /* token location, or -1 if unknown */
+};
+
+/*
+ * JsonTable -
+ * untransformed representation of JSON_TABLE
+ */
+typedef struct JsonTable
+{
+ NodeTag type;
+ JsonCommon *common; /* common JSON path syntax fields */
+ List *columns; /* list of JsonTableColumn */
+ JsonTablePlan *plan; /* join plan, if specified */
+ JsonBehavior *on_error; /* ON ERROR behavior, if specified */
+ Alias *alias; /* table alias in FROM clause */
+ bool lateral; /* does it have LATERAL prefix? */
+ int location; /* token location, or -1 if unknown */
+} JsonTable;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 8f3723ef4c..f026bb732e 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -86,8 +86,14 @@ typedef struct RangeVar
int location;
} RangeVar;
+typedef enum TableFuncType
+{
+ TFT_XMLTABLE,
+ TFT_JSON_TABLE
+} TableFuncType;
+
/*
- * TableFunc - node for a table function, such as XMLTABLE.
+ * TableFunc - node for a table function, such as XMLTABLE or JSON_TABLE.
*
* Entries in the ns_names list are either String nodes containing
* literal namespace names, or NULL pointers to represent DEFAULT.
@@ -95,6 +101,8 @@ typedef struct RangeVar
typedef struct TableFunc
{
NodeTag type;
+ /* XMLTABLE or JSON_TABLE */
+ TableFuncType functype;
/* list of namespace URI expressions */
List *ns_uris pg_node_attr(query_jumble_ignore);
/* list of namespace names or NULL */
@@ -115,8 +123,14 @@ typedef struct TableFunc
List *colexprs;
/* list of column default expressions */
List *coldefexprs pg_node_attr(query_jumble_ignore);
+ /* list of column value expressions */
+ List *colvalexprs pg_node_attr(query_jumble_ignore);
+ /* list of PASSING argument expressions */
+ List *passingvalexprs pg_node_attr(query_jumble_ignore);
/* nullability flag for each output column */
Bitmapset *notnulls pg_node_attr(query_jumble_ignore);
+ /* JSON_TABLE plan */
+ Node *plan pg_node_attr(query_jumble_ignore);
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1552,7 +1566,8 @@ typedef enum JsonExprOp
{
JSON_VALUE_OP, /* JSON_VALUE() */
JSON_QUERY_OP, /* JSON_QUERY() */
- JSON_EXISTS_OP /* JSON_EXISTS() */
+ JSON_EXISTS_OP, /* JSON_EXISTS() */
+ JSON_TABLE_OP /* JSON_TABLE() */
} JsonExprOp;
/*
@@ -1777,6 +1792,48 @@ typedef struct JsonExpr
int location; /* token location, or -1 if unknown */
} JsonExpr;
+/*
+ * JsonTablePath
+ * A JSON path expression to be computed as part of evaluating
+ * a JSON_TABLE plan node
+ */
+typedef struct JsonTablePath
+{
+ NodeTag type;
+
+ Const *value;
+ char *name;
+} JsonTablePath;
+
+/*
+ * JsonTableParent -
+ * transformed representation of parent JSON_TABLE plan node
+ */
+typedef struct JsonTableParent
+{
+ NodeTag type;
+ JsonTablePath *path;
+ Node *child; /* nested columns, if any */
+ bool outerJoin; /* outer or inner join for nested columns? */
+ int colMin; /* min column index in the resulting column
+ * list */
+ int colMax; /* max column index in the resulting column
+ * list */
+ bool errorOnError; /* ERROR/EMPTY ON ERROR behavior */
+} JsonTableParent;
+
+/*
+ * JsonTableSibling -
+ * transformed representation of joined sibling JSON_TABLE plan node
+ */
+typedef struct JsonTableSibling
+{
+ NodeTag type;
+ Node *larg; /* left join node */
+ Node *rarg; /* right join node */
+ bool cross; /* cross or union join? */
+} JsonTableSibling;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 0954d9fc7b..f88a6c9ac6 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -242,6 +242,7 @@ PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_table", JSON_TABLE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -284,6 +285,7 @@ PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("nested", NESTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -334,7 +336,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_clause.h b/src/include/parser/parse_clause.h
index 8e26482db4..7152a8ef48 100644
--- a/src/include/parser/parse_clause.h
+++ b/src/include/parser/parse_clause.h
@@ -51,4 +51,7 @@ extern List *addTargetToSortList(ParseState *pstate, TargetEntry *tle,
extern Index assignSortGroupRef(TargetEntry *tle, List *tlist);
extern bool targetIsInSortList(TargetEntry *tle, Oid sortop, List *sortList);
+/* functions in parse_jsontable.c */
+extern ParseNamespaceItem *transformJsonTable(ParseState *pstate, JsonTable *jt);
+
#endif /* PARSE_CLAUSE_H */
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index 5a37133847..7eb0ccf4e4 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -15,6 +15,7 @@
#define JSONPATH_H
#include "fmgr.h"
+#include "executor/tablefunc.h"
#include "nodes/pg_list.h"
#include "nodes/primnodes.h"
#include "utils/jsonb.h"
@@ -288,4 +289,6 @@ extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
bool *error, List *vars);
+extern PGDLLIMPORT const TableFuncRoutine JsonbTableRoutine;
+
#endif
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
index 8b87580752..995f267404 100644
--- a/src/test/regress/expected/json_sqljson.out
+++ b/src/test/regress/expected/json_sqljson.out
@@ -16,3 +16,9 @@ ERROR: JSON_QUERY() is not yet implemented for the json type
LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
^
HINT: Try casting the argument to jsonb
+-- JSON_TABLE
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
+ERROR: JSON_TABLE() is not yet implemented for the json type
+LINE 1: SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo ...
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
index 8dcdb90fc8..a3ba44cf01 100644
--- a/src/test/regress/expected/jsonb_sqljson.out
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -1069,3 +1069,1189 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.dateti
ERROR: functions in index expression must be marked IMMUTABLE
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+-- JSON_TABLE
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+ERROR: syntax error at or near "("
+LINE 1: SELECT JSON_TABLE('[]', '$');
+ ^
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ERROR: syntax error at or near ")"
+LINE 1: SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ ^
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+ERROR: JSON_TABLE function has 1 columns available but 2 columns specified
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+ foo
+-----
+(0 rows)
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+ item | foo
+------+-----
+ 123 |
+(1 row)
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | int | text | char(4) | bool | numeric | domain | js | jb
+---------------------------------------------------------------------------------------+----+-----+---------+---------+------+---------+---------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [] | | | | | | | | |
+ {} | 1 | | | | | | | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1 | 1.23 | 1.23 | | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | 2 | 2 | 2 | | 2 | 2 | "2" | "2"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | aaaaaaa | aaaa | | | aaaaaaa | "aaaaaaa" | "aaaaaaa"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | foo | foo | | | | "foo" | "foo"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | | | | | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | 0 | false | fals | f | | false | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | 1 | true | true | t | | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | | | | | | | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | [1,2] | [1,2 | | | [1,2] | "[1,2]" | "[1,2]"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | "str" | "str | | | "str" | "\"str\"" | "\"str\""
+(14 rows)
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+ js | id | jst | jsc | jsv | jsb | jsbq
+---------------------------------------------------------------------------------------+----+--------------+------+------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | 1 | 1
+ [] | | | | | |
+ {} | 1 | {} | {} | {} | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1.23 | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | "2" | "2" | "2" | "2" | 2
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | "aaaaaaa" | "aaa | "aaa | "aaaaaaa" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | "foo" | "foo | "foo | "foo" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | null | null | null | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | false | fals | fals | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | true | true | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | {"aaa": 123} | {"aa | {"aa | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | "[1,2]" | "[1, | "[1, | "[1,2]" | [1, 2]
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | "\"str\"" | "\"s | "\"s | "\"str\"" | "str"
+(14 rows)
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+ js | id | exists1 | exists2 | exists3 | exists4
+---------------------------------------------------------------------------------------+----+---------+---------+---------+---------
+ 1 | 1 | f | 0 | | false
+ [] | | | | |
+ {} | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | t | 1 | 1 | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | f | 0 | | false
+(14 rows)
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | aaa | aaa1 | js2 | jsb2w | jsb2q | ia | ta | jba
+---------------------------------------------------------------------------------------+----+-----+------+--------------+----------------+--------------+----+----+-----
+ 1 | 1 | | | 1 | [1] | 1 | | |
+ [] | | | | | | | | |
+ {} | 1 | | | {} | [{}] | {} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | | | 1 | [1] | 1 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | | | 1.23 | [1.23] | 1.23 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | | | "2" | ["2"] | 2 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | | "aaaaaaa" | ["aaaaaaa"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | | "foo" | ["foo"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | null | [null] | null | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | | | false | [false] | false | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | | | true | [true] | true | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | 123 | 123 | {"aaa": 123} | [{"aaa": 123}] | {"aaa": 123} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | | "[1,2]" | ["[1,2]"] | [1, 2] | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | | "\"str\"" | ["\"str\""] | "str" | | |
+(14 rows)
+
+-- JSON_TABLE: Test backward parsing
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+\sv jsonb_table_view
+CREATE OR REPLACE VIEW public.jsonb_table_view AS
+ SELECT id,
+ "int",
+ text,
+ "char(4)",
+ bool,
+ "numeric",
+ domain,
+ js,
+ jb,
+ jst,
+ jsc,
+ jsv,
+ jsb,
+ jsbq,
+ aaa,
+ aaa1,
+ exists1,
+ exists2,
+ exists3,
+ js2,
+ jsb2w,
+ jsb2q,
+ ia,
+ ta,
+ jba,
+ a1,
+ b1,
+ a11,
+ a21,
+ a22
+ FROM JSON_TABLE(
+ 'null'::jsonb, '$[*]' AS json_table_path_1
+ PASSING
+ 1 + 2 AS a,
+ '"foo"'::json AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" integer PATH '$',
+ text text PATH '$',
+ "char(4)" character(4) PATH '$',
+ bool boolean PATH '$',
+ "numeric" numeric PATH '$',
+ domain jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc character(4) FORMAT JSON PATH '$',
+ jsv character varying(4) FORMAT JSON PATH '$',
+ jsb jsonb PATH '$',
+ jsbq jsonb PATH '$' OMIT QUOTES,
+ aaa integer PATH '$."aaa"',
+ aaa1 integer PATH '$."aaa"',
+ exists1 boolean EXISTS PATH '$."aaa"',
+ exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia integer[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1
+ COLUMNS (
+ a1 integer PATH '$."a1"',
+ b1 text PATH '$."b1"',
+ NESTED PATH '$[*]' AS "p1 1"
+ COLUMNS (
+ a11 text PATH '$."a11"'
+ )
+ ),
+ NESTED PATH '$[2]' AS p2
+ COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1"
+ COLUMNS (
+ a21 text PATH '$."a21"'
+ ),
+ NESTED PATH '$[*]' AS p22
+ COLUMNS (
+ a22 text PATH '$."a22"'
+ )
+ )
+ )
+ PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22))))
+ )
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+ QUERY PLAN
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ Table Function Scan on "json_table"
+ Output: "json_table".id, "json_table"."int", "json_table".text, "json_table"."char(4)", "json_table".bool, "json_table"."numeric", "json_table".domain, "json_table".js, "json_table".jb, "json_table".jst, "json_table".jsc, "json_table".jsv, "json_table".jsb, "json_table".jsbq, "json_table".aaa, "json_table".aaa1, "json_table".exists1, "json_table".exists2, "json_table".exists3, "json_table".js2, "json_table".jsb2w, "json_table".jsb2q, "json_table".ia, "json_table".ta, "json_table".jba, "json_table".a1, "json_table".b1, "json_table".a11, "json_table".a21, "json_table".a22
+ Table Function Call: JSON_TABLE('null'::jsonb, '$[*]' AS json_table_path_1 PASSING 3 AS a, '"foo"'::jsonb AS "b c" COLUMNS (id FOR ORDINALITY, "int" integer PATH '$', text text PATH '$', "char(4)" character(4) PATH '$', bool boolean PATH '$', "numeric" numeric PATH '$', domain jsonb_test_domain PATH '$', js json PATH '$', jb jsonb PATH '$', jst text FORMAT JSON PATH '$', jsc character(4) FORMAT JSON PATH '$', jsv character varying(4) FORMAT JSON PATH '$', jsb jsonb PATH '$', jsbq jsonb PATH '$' OMIT QUOTES, aaa integer PATH '$."aaa"', aaa1 integer PATH '$."aaa"', exists1 boolean EXISTS PATH '$."aaa"', exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR, exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR, js2 json PATH '$', jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER, jsb2q jsonb PATH '$' OMIT QUOTES, ia integer[] PATH '$', ta text[] PATH '$', jba jsonb[] PATH '$', NESTED PATH '$[1]' AS p1 COLUMNS (a1 integer PATH '$."a1"', b1 text PATH '$."b1"', NESTED PATH '$[*]' AS "p1 1" COLUMNS (a11 text PATH '$."a11"')), NESTED PATH '$[2]' AS p2 COLUMNS ( NESTED PATH '$[*]' AS "p2:1" COLUMNS (a21 text PATH '$."a21"'), NESTED PATH '$[*]' AS p22 COLUMNS (a22 text PATH '$."a22"'))) PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22)))))
+(3 rows)
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+ERROR: cannot use more than one FOR ORDINALITY column
+LINE 1: ..._TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR OR...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+ id | a
+----+---
+ 1 | 1
+(1 row)
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+ js | a
+-------+---
+ 1 | 1
+ "err" |
+(2 rows)
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+ a
+---
+
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: no SQL/JSON item
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 1
+(1 row)
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+ a
+---
+ 0
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to smallint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to bigint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to real
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 E...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+ a
+-----
+ fal
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to json
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to jsonb
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EX...
+ ^
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+ERROR: cannot use OMIT/KEEP QUOTES clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+ item
+---------
+ "world"
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+ERROR: cannot use WITH WRAPPER clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+ item
+-----------
+ ["world"]
+(1 row)
+
+-- JSON_TABLE: nested paths and plans
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb '[]', '$' -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 4: NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: b
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+-- JSON_TABLE: plan validation
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p1)
+ ^
+DETAIL: Path name mismatch: expected p0 but p1 is given.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 UNION p1 UNION p11)
+ ^
+DETAIL: Expected INNER or OUTER JSON_TABLE plan node.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 8: NESTED PATH '$' AS p2 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p2 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 5: NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ ^
+DETAIL: Plan node for nested path p11 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+ ^
+DETAIL: Plan node contains some extra or duplicate sibling nodes.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 6: NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ^
+DETAIL: Plan node for nested path p12 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 9: NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ ^
+DETAIL: Plan node for nested path p21 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+ bar | foo | baz
+-----+-----+-----
+(0 rows)
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb 'null', 'strict $[*]' -- without root path name
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- JSON_TABLE: plan execution
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+ n | a | c | b
+---+----+----+---
+ 1 | 1 | |
+ 2 | 2 | 10 |
+ 2 | 2 | |
+ 2 | 2 | 20 |
+ 2 | 2 | | 1
+ 2 | 2 | | 2
+ 2 | 2 | | 3
+ 3 | 3 | | 1
+ 3 | 3 | | 2
+ 4 | -1 | | 1
+ 4 | -1 | | 2
+(11 rows)
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+ n | a | b | b1 | c | c1 | b
+---+---+--------------+-----+------+----+-----
+ 1 | 1 | [1, 10] | 1 | 1 | | 101
+ 1 | 1 | [1, 10] | 1 | null | | 101
+ 1 | 1 | [1, 10] | 1 | 2 | | 101
+ 1 | 1 | [1, 10] | 10 | 1 | | 110
+ 1 | 1 | [1, 10] | 10 | null | | 110
+ 1 | 1 | [1, 10] | 10 | 2 | | 110
+ 1 | 1 | [2] | 2 | 1 | | 102
+ 1 | 1 | [2] | 2 | null | | 102
+ 1 | 1 | [2] | 2 | 2 | | 102
+ 1 | 1 | [3, 30, 300] | 3 | 1 | | 103
+ 1 | 1 | [3, 30, 300] | 3 | null | | 103
+ 1 | 1 | [3, 30, 300] | 3 | 2 | | 103
+ 1 | 1 | [3, 30, 300] | 30 | 1 | | 130
+ 1 | 1 | [3, 30, 300] | 30 | null | | 130
+ 1 | 1 | [3, 30, 300] | 30 | 2 | | 130
+ 1 | 1 | [3, 30, 300] | 300 | 1 | | 400
+ 1 | 1 | [3, 30, 300] | 300 | null | | 400
+ 1 | 1 | [3, 30, 300] | 300 | 2 | | 400
+ 2 | 2 | | | | |
+ 3 | | | | | |
+(20 rows)
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+ x | y | y | z
+---+---+--------------+---
+ 2 | 1 | [1, 2, 3] | 1
+ 2 | 1 | [1, 2, 3] | 2
+ 2 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [1, 2, 3] | 1
+ 3 | 1 | [1, 2, 3] | 2
+ 3 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [2, 3, 4, 5] | 2
+ 3 | 1 | [2, 3, 4, 5] | 3
+ 3 | 1 | [2, 3, 4, 5] | 4
+ 3 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [1, 2, 3] | 1
+ 4 | 1 | [1, 2, 3] | 2
+ 4 | 1 | [1, 2, 3] | 3
+ 4 | 1 | [2, 3, 4, 5] | 2
+ 4 | 1 | [2, 3, 4, 5] | 3
+ 4 | 1 | [2, 3, 4, 5] | 4
+ 4 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [3, 4, 5, 6] | 3
+ 4 | 1 | [3, 4, 5, 6] | 4
+ 4 | 1 | [3, 4, 5, 6] | 5
+ 4 | 1 | [3, 4, 5, 6] | 6
+ 2 | 2 | [1, 2, 3] | 2
+ 2 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [1, 2, 3] | 2
+ 3 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [2, 3, 4, 5] | 2
+ 3 | 2 | [2, 3, 4, 5] | 3
+ 3 | 2 | [2, 3, 4, 5] | 4
+ 3 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [1, 2, 3] | 2
+ 4 | 2 | [1, 2, 3] | 3
+ 4 | 2 | [2, 3, 4, 5] | 2
+ 4 | 2 | [2, 3, 4, 5] | 3
+ 4 | 2 | [2, 3, 4, 5] | 4
+ 4 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [3, 4, 5, 6] | 3
+ 4 | 2 | [3, 4, 5, 6] | 4
+ 4 | 2 | [3, 4, 5, 6] | 5
+ 4 | 2 | [3, 4, 5, 6] | 6
+ 2 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [2, 3, 4, 5] | 3
+ 3 | 3 | [2, 3, 4, 5] | 4
+ 3 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [1, 2, 3] | 3
+ 4 | 3 | [2, 3, 4, 5] | 3
+ 4 | 3 | [2, 3, 4, 5] | 4
+ 4 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [3, 4, 5, 6] | 3
+ 4 | 3 | [3, 4, 5, 6] | 4
+ 4 | 3 | [3, 4, 5, 6] | 5
+ 4 | 3 | [3, 4, 5, 6] | 6
+(52 rows)
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+ERROR: could not find jsonpath variable "x"
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+ json_value
+------------
+ foo
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_query
+------------
+ 123
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+ json_query
+------------
+ [123]
+(1 row)
+
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+ERROR: syntax error at or near " " of jsonpath input
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
+ERROR: only string constants supported in JSON_TABLE path specification
+LINE 1: SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || '...
+ ^
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
index 4f30fa46b9..df4a430d88 100644
--- a/src/test/regress/sql/json_sqljson.sql
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -9,3 +9,7 @@ SELECT JSON_VALUE(NULL FORMAT JSON, '$');
-- JSON_QUERY
SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+
+-- JSON_TABLE
+
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
index 6c3f8c7e43..ab73a01130 100644
--- a/src/test/regress/sql/jsonb_sqljson.sql
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -339,3 +339,677 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime()
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+
+-- JSON_TABLE
+
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- JSON_TABLE: Test backward parsing
+
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+
+\sv jsonb_table_view
+
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+
+-- JSON_TABLE: nested paths and plans
+
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+
+-- JSON_TABLE: plan validation
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+
+-- JSON_TABLE: plan execution
+
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 6f45542836..453e4e446e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1310,6 +1310,7 @@ JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
+JsonPathSpec
JsonPathString
JsonPathVarCallback
JsonPathVariable
@@ -1319,6 +1320,17 @@ JsonReturning
JsonScalarExpr
JsonSemAction
JsonSerializeExpr
+JsonTable
+JsonTableColumn
+JsonTableColumnType
+JsonTableContext
+JsonTableJoinState
+JsonTableParent
+JsonTablePlan
+JsonTablePlanJoinType
+JsonTablePlanType
+JsonTableScanState
+JsonTableSibling
JsonTokenType
JsonTransformStringValuesAction
JsonTypeCategory
@@ -2775,6 +2787,7 @@ TableFunc
TableFuncRoutine
TableFuncScan
TableFuncScanState
+TableFuncType
TableInfo
TableLikeClause
TableSampleClause
--
2.35.3
[application/octet-stream] v22-0003-SQL-JSON-query-functions.patch (203.5K, ../../CA+HiwqETtgH0ESDEEGDwTmazmtNeR7UsJS=BOoa=ghdYMo+Ufg@mail.gmail.com/6-v22-0003-SQL-JSON-query-functions.patch)
download | inline diff:
From 99d1b96d2b30c204f34d2a05d61e2e4178d1b77b Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:12 +0900
Subject: [PATCH v22 3/5] SQL/JSON query functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This introduces the SQL/JSON functions for querying JSON data using
jsonpath expressions. The functions are:
JSON_EXISTS()
JSON_QUERY()
JSON_VALUE()
JSON_EXISTS() tests if the jsonpath expression applied to the jsonb
value yields any values.
JSON_VALUE() must return a single value, and an error occurs if it
tries to return multiple values.
JSON_QUERY() must return a json object or array, and there are
various WRAPPER options for handling scalar or multi-value results.
Both these functions have options for handling EMPTY and ERROR
conditions.
All of these functions only operate on jsonb. The workaround for now
is to cast the argument to jsonb.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 148 +++
src/backend/executor/execExpr.c | 455 ++++++++
src/backend/executor/execExprInterp.c | 545 +++++++++-
src/backend/jit/llvm/llvmjit_expr.c | 265 +++++
src/backend/jit/llvm/llvmjit_types.c | 5 +
src/backend/nodes/makefuncs.c | 16 +
src/backend/nodes/nodeFuncs.c | 150 +++
src/backend/optimizer/path/costsize.c | 3 +-
src/backend/optimizer/util/clauses.c | 19 +
src/backend/parser/gram.y | 348 +++++-
src/backend/parser/parse_expr.c | 543 +++++++++-
src/backend/parser/parse_target.c | 15 +
src/backend/utils/adt/formatting.c | 44 +
src/backend/utils/adt/jsonb.c | 62 ++
src/backend/utils/adt/jsonfuncs.c | 52 +-
src/backend/utils/adt/jsonpath.c | 255 +++++
src/backend/utils/adt/jsonpath_exec.c | 391 ++++++-
src/backend/utils/adt/ruleutils.c | 137 +++
src/include/executor/execExpr.h | 145 +++
src/include/fmgr.h | 1 +
src/include/nodes/makefuncs.h | 1 +
src/include/nodes/parsenodes.h | 59 +
src/include/nodes/primnodes.h | 115 ++
src/include/parser/kwlist.h | 11 +
src/include/utils/formatting.h | 1 +
src/include/utils/jsonb.h | 3 +
src/include/utils/jsonfuncs.h | 5 +
src/include/utils/jsonpath.h | 27 +
src/interfaces/ecpg/preproc/ecpg.trailer | 28 +
src/test/regress/expected/json_sqljson.out | 18 +
src/test/regress/expected/jsonb_sqljson.out | 1071 +++++++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/json_sqljson.sql | 11 +
src/test/regress/sql/jsonb_sqljson.sql | 341 ++++++
src/tools/pgindent/typedefs.list | 20 +
35 files changed, 5244 insertions(+), 68 deletions(-)
create mode 100644 src/test/regress/expected/json_sqljson.out
create mode 100644 src/test/regress/expected/jsonb_sqljson.out
create mode 100644 src/test/regress/sql/json_sqljson.sql
create mode 100644 src/test/regress/sql/jsonb_sqljson.sql
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 24ad87f910..ddc4f4f6aa 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17056,6 +17056,154 @@ array w/o UK? | t
</tbody>
</tgroup>
</table>
+
+ <para>
+ <xref linkend="functions-sqljson-querying"/> details the SQL/JSON
+ functions that can be used to query JSON data.
+ </para>
+
+ <note>
+ <para>
+ SQL/JSON paths can only be applied to the <type>jsonb</type> type, so it
+ might be necessary to cast the <replaceable>context_item</replaceable>
+ argument of these functions to <type>jsonb</type>.
+ </para>
+ </note>
+
+ <table id="functions-sqljson-querying">
+ <title>SQL/JSON Query Functions</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ Function signature
+ </para>
+ <para>
+ Description
+ </para>
+ <para>
+ Example(s)
+ </para></entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_exists</primary></indexterm>
+ <function>json_exists</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> </optional>
+ <optional> { <literal>TRUE</literal> | <literal>FALSE</literal> |<literal> UNKNOWN</literal> | <literal>ERROR</literal> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns true if the SQL/JSON <replaceable>path_expression</replaceable>
+ applied to the <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s yields any items.
+ The <literal>ON ERROR</literal> clause specifies what is returned if
+ an error occurs; the default is to return <literal>FALSE</literal>.
+ Note that if the <replaceable>path_expression</replaceable>
+ is <literal>strict</literal>, an error is generated if it yields no items.
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"key1": [1,2,3]}', 'strict $.key1[*] ? (@ > 2)')</literal>
+ <returnvalue>t</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'lax $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>f</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'strict $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>ERROR: jsonpath array subscript is out of bounds</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_query</primary></indexterm>
+ <function>json_query</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> <optional> <literal>FORMAT JSON</literal> <optional> <literal>ENCODING UTF8</literal> </optional> </optional> </optional>
+ <optional> { <literal>WITHOUT</literal> | <literal>WITH</literal> { <literal>CONDITIONAL</literal> | <optional><literal>UNCONDITIONAL</literal></optional> } } <optional> <literal>ARRAY</literal> </optional> <literal>WRAPPER</literal> </optional>
+ <optional> { <literal>KEEP</literal> | <literal>OMIT</literal> } <literal>QUOTES</literal> <optional> <literal>ON SCALAR STRING</literal> </optional> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s.
+ This function must return a JSON string, so if the path expression
+ returns multiple SQL/JSON items, you must wrap the result using the
+ <literal>WITH WRAPPER</literal> clause. If the wrapper is
+ <literal>UNCONDITIONAL</literal>, an array wrapper will always
+ be applied, even if the returned value is already a single JSON object
+ or array, but if it is <literal>CONDITIONAL</literal>, it will not be
+ applied to a single array or object. <literal>UNCONDITIONAL</literal>
+ is the default. If the result is a scalar string, by default the value
+ returned will have surrounding quotes making it a valid JSON value,
+ which can be made explicit by specifying <literal>KEEP QUOTES</literal>.
+ Conversely, quotes can be omitted by specifying <literal>OMIT QUOTES</literal>.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>;
+ the default returned type is <type>jsonb</type>.
+ The <literal>ON EMPTY</literal> clause specifies the behavior if the
+ <replaceable>path_expression</replaceable> yields no value at all; the
+ default when <literal>ON EMPTY</literal> is not specified is to return a
+ null value. The <literal>ON ERROR</literal> clause specifies the behavior
+ if an error occurs as a result of <type>jsonpath</type> evaluation
+ (including cast to the output type) or during the execution of
+ <literal>ON EMPTY</literal> behavior (that was caused by empty result of
+ <type>jsonpath</type> evaluation); the default when
+ <literal>ON ERROR</literal> is not specified is to return a null value.
+ </para>
+ <para>
+ <literal>json_query(jsonb '[1,[2,3],null]', 'lax $[*][1]' WITH CONDITIONAL WRAPPER)</literal>
+ <returnvalue>[3]</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_value</primary></indexterm>
+ <function>json_value</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable>
+ <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <literal>PASSING</literal> <replaceable>value</replaceable>s. The
+ extracted value must be a single <acronym>SQL/JSON</acronym> scalar
+ item. For results that are objects or arrays, use the
+ <function>json_query</function> function instead.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>.
+ The default returned type is <type>text</type>.
+ The <literal>ON ERROR</literal> and <literal>ON EMPTY</literal>
+ clauses have similar semantics as mentioned in the description of
+ <function>json_query</function>.
+ </para>
+ <para>
+ <literal>json_value(jsonb '"123.45"', '$' RETURNING float)</literal>
+ <returnvalue>123.45</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '"03:04 2015-02-01"', '$.datetime("HH24:MI YYYY-MM-DD")' RETURNING date)</literal>
+ <returnvalue>2015-02-01</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '[1,2]', 'strict $[*]' DEFAULT 9 ON ERROR)</literal>
+ <returnvalue>9</returnvalue>
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
</sect2>
<sect2 id="functions-sqljson-path">
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 78796acd8f..e8bc2cb02c 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -49,6 +49,7 @@
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/typcache.h"
@@ -88,6 +89,18 @@ static void ExecBuildAggTransCall(ExprState *state, AggState *aggstate,
FunctionCallInfo fcinfo, AggStatePerTrans pertrans,
int transno, int setno, int setoff, bool ishash,
bool nullcheck);
+static void ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch);
+static Datum GetJsonBehaviorConstVal(JsonBehavior *behavior, bool *is_null);
+static JsonCoercionState *ExecInitJsonCoercion(ExprEvalStep *scratch, ExprState *state,
+ JsonCoercion *coercion,
+ JsonBehavior *on_error,
+ Datum *resv, bool *resnull);
+static List *ExecInitJsonItemCoercions(ExprEvalStep *scratch, ExprState *state,
+ List *item_coercions,
+ JsonBehavior *on_error,
+ Datum *resv, bool *resnull);
/*
@@ -2403,6 +2416,14 @@ ExecInitExprRec(Expr *node, ExprState *state,
break;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+
+ ExecInitJsonExpr(jexpr, state, resv, resnull, &scratch);
+ break;
+ }
+
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -4172,3 +4193,437 @@ ExecBuildParamSetEqual(TupleDesc desc,
return state;
}
+
+/*
+ * Push steps to evaluate a JsonExpr and its various subsidiary expressions.
+ */
+static void
+ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch)
+{
+ JsonExprState *jsestate = palloc0(sizeof(JsonExprState));
+ JsonExprPreEvalState *pre_eval = &jsestate->pre_eval;
+ ListCell *argexprlc;
+ ListCell *argnamelc;
+ int skip_step_off = -1;
+ int passing_args_step_off = -1;
+ int coercion_step_off = -1;
+ int coercion_finish_step_off = -1;
+ int behavior_step_off = -1;
+ int onempty_expr_step_off = -1;
+ int onempty_jump_step_off = -1;
+ int onerror_expr_step_off = -1;
+ int onerror_jump_step_off = -1;
+ int result_coercion_jump_step_off = -1;
+ List *adjust_jumps = NIL;
+ ListCell *lc;
+ ExprEvalStep *as;
+
+ jsestate->jsexpr = jexpr;
+
+ /*
+ * Add steps to compute formatted_expr, pathspec, and PASSING arg
+ * expressions as things that must be evaluated *before* the actual JSON
+ * path expression.
+ */
+ ExecInitExprRec((Expr *) jexpr->formatted_expr, state,
+ &pre_eval->formatted_expr.value,
+ &pre_eval->formatted_expr.isnull);
+ ExecInitExprRec((Expr *) jexpr->path_spec, state,
+ &pre_eval->pathspec.value,
+ &pre_eval->pathspec.isnull);
+
+ /*
+ * Before pushing steps for PASSING args, push a step to decide whether to
+ * skip evaluating the args and the JSON path expression depending on
+ * whether either of formatted_expr and pathspec is NULL; see
+ * ExecEvalJsonExprSkip().
+ */
+ scratch->opcode = EEOP_JSONEXPR_SKIP;
+ scratch->d.jsonexpr_skip.jsestate = jsestate;
+ skip_step_off = state->steps_len;
+ ExprEvalPushStep(state, scratch);
+
+ /* PASSING args. */
+ jsestate->pre_eval.args = NIL;
+ passing_args_step_off = state->steps_len;
+ forboth(argexprlc, jexpr->passing_values,
+ argnamelc, jexpr->passing_names)
+ {
+ Expr *argexpr = (Expr *) lfirst(argexprlc);
+ String *argname = lfirst_node(String, argnamelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = pstrdup(argname->sval);
+ var->typid = exprType((Node *) argexpr);
+ var->typmod = exprTypmod((Node *) argexpr);
+
+ ExecInitExprRec((Expr *) argexpr, state, &var->value, &var->isnull);
+
+ pre_eval->args = lappend(pre_eval->args, var);
+ }
+
+ /* Step for the actual JSON path evaluation; see ExecEvalJsonExpr(). */
+ scratch->opcode = EEOP_JSONEXPR_PATH;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Step to handle ON ERROR and ON EMPTY behavior. Also, to handle errors
+ * that may occur during coercion handling.
+ *
+ * See ExecEvalJsonExprBehavior().
+ */
+ scratch->opcode = EEOP_JSONEXPR_BEHAVIOR;
+ scratch->d.jsonexpr_behavior.jsestate = jsestate;
+ behavior_step_off = state->steps_len;
+ ExprEvalPushStep(state, scratch);
+
+ /* Step(s) to evaluate the ON EMPTY expression */
+ onempty_expr_step_off = state->steps_len;
+ if (jexpr->on_empty &&
+ jexpr->on_empty->btype != JSON_BEHAVIOR_ERROR)
+ {
+ if (jexpr->on_empty->default_expr)
+ {
+ ExecInitExprRec((Expr *) jexpr->on_empty->default_expr,
+ state, resv, resnull);
+
+ /*
+ * Emit JUMP step to jump to the step after JsonExpr steps,
+ * because the default expression has already been coerced, so
+ * there's nothing more to do.
+ */
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ ExprEvalPushStep(state, scratch);
+ adjust_jumps = lappend_int(adjust_jumps, state->steps_len - 1);
+ }
+ else
+ {
+ Datum constvalue;
+ bool constisnull;
+
+ constvalue = GetJsonBehaviorConstVal(jexpr->on_empty,
+ &constisnull);
+ scratch->opcode = EEOP_CONST;
+ scratch->d.constval.value = constvalue;
+ scratch->d.constval.isnull = constisnull;
+
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Emit JUMP step to jump to the coercion step to coerce the above
+ * value to the desired output type.
+ */
+ onempty_jump_step_off = state->steps_len;
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /* Step(s) to evaluate the ON ERROR expression */
+ onerror_expr_step_off = state->steps_len;
+ if (jexpr->on_error &&
+ jexpr->on_error->btype != JSON_BEHAVIOR_ERROR)
+ {
+ if (jexpr->on_error->default_expr)
+ {
+ ExecInitExprRec((Expr *) jexpr->on_error->default_expr,
+ state, resv, resnull);
+
+ /*
+ * Emit JUMP step to jump to the step after JsonExpr steps,
+ * because the default expression has already been coerced, so
+ * there's nothing more to do.
+ */
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ ExprEvalPushStep(state, scratch);
+ adjust_jumps = lappend_int(adjust_jumps, state->steps_len - 1);
+ }
+ else
+ {
+ Datum constvalue;
+ bool constisnull;
+
+ constvalue = GetJsonBehaviorConstVal(jexpr->on_error,
+ &constisnull);
+ scratch->opcode = EEOP_CONST;
+ scratch->d.constval.value = constvalue;
+ scratch->d.constval.isnull = constisnull;
+
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Emit JUMP step to jump to the coercion step to coerce the above
+ * value to the desired output type.
+ */
+ onerror_jump_step_off = state->steps_len;
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set later */
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /*
+ * Step to handle applying coercion to the JSON item returned by
+ * ExecEvalJsonExpr() or to the ON EMPTY/ERROR expression as
+ * ExecEvalJsonExprBehavior() decides.
+ *
+ * See ExecEvalJsonExprCoercion().
+ */
+ scratch->opcode = EEOP_JSONEXPR_COERCION;
+ scratch->d.jsonexpr_coercion.jsestate = jsestate;
+ coercion_step_off = state->steps_len;
+ ExprEvalPushStep(state, scratch);
+
+ /* Initialize coercion expression(s). */
+ if (jexpr->result_coercion)
+ {
+ jsestate->result_jcstate =
+ ExecInitJsonCoercion(scratch, state, jexpr->result_coercion,
+ jexpr->on_error, resv, resnull);
+ /* Emit JUMP step to jump to the step after JsonExpr steps. */
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ result_coercion_jump_step_off = state->steps_len;
+ ExprEvalPushStep(state, scratch);
+ }
+ if (jexpr->item_coercions)
+ {
+ /*
+ * ExecPrepareJsonItemCoercion() called by ExecEvalJsonExpr() chooses
+ * one for a given JSON item returned by JsonPathValue().
+ */
+ jsestate->item_jcstates =
+ ExecInitJsonItemCoercions(scratch, state, jexpr->item_coercions,
+ jexpr->on_error, resv, resnull);
+ }
+
+ /*
+ * And a step to clean up after the coercion step; see
+ * ExecEvalJsonExprCoercionFinish().
+ */
+ scratch->opcode = EEOP_JSONEXPR_COERCION_FINISH;
+ scratch->d.jsonexpr_coercion.jsestate = jsestate;
+ coercion_finish_step_off = state->steps_len;
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Adjust jump target addresses in various post-eval steps now that we
+ * have all the steps in place.
+ */
+
+ /* EEOP_JSONEXPR_SKIP */
+ Assert(skip_step_off >= 0);
+ as = &state->steps[skip_step_off];
+ as->d.jsonexpr_skip.jump_coercion = coercion_step_off;
+ as->d.jsonexpr_skip.jump_passing_args = passing_args_step_off;
+
+ /* EEOP_JSONEXPR_BEHAVIOR */
+ Assert(behavior_step_off >= 0);
+ as = &state->steps[behavior_step_off];
+ as->d.jsonexpr_behavior.jump_onerror_expr = onerror_expr_step_off;
+ as->d.jsonexpr_behavior.jump_onempty_expr = onempty_expr_step_off;
+ as->d.jsonexpr_behavior.jump_coercion = coercion_step_off;
+ as->d.jsonexpr_behavior.jump_skip_coercion = coercion_finish_step_off + 1;
+
+ /* EEOP_JSONEXPR_COERCION */
+ Assert(coercion_step_off >= 0);
+ as = &state->steps[coercion_step_off];
+ as->d.jsonexpr_coercion.jump_coercion_error = behavior_step_off;
+ as->d.jsonexpr_coercion.jump_coercion_done = coercion_finish_step_off + 1;
+
+ /* EEOP_JSONEXPR_COERCION_FINISH */
+ Assert(coercion_finish_step_off >= 0);
+ as = &state->steps[coercion_finish_step_off];
+ as->d.jsonexpr_coercion_finish.jump_coercion_error = behavior_step_off;
+ as->d.jsonexpr_coercion_finish.jump_coercion_done = coercion_finish_step_off + 1;
+
+ /* EEOP_JUMP steps */
+
+ /*
+ * Ones after ON EMPTY and ON ERROR non-default expressions should jump to
+ * the coercion step.
+ */
+ if (onempty_jump_step_off >= 0)
+ {
+ as = &state->steps[onempty_jump_step_off];
+ as->d.jump.jumpdone = coercion_step_off;
+ }
+ if (onerror_jump_step_off >= 0)
+ {
+ as = &state->steps[onerror_jump_step_off];
+ as->d.jump.jumpdone = coercion_step_off;
+ }
+
+ /*
+ * Jump to EEOP_JSONEXPR_COERCION_FINISH after evaluating result_coercion.
+ */
+ if (result_coercion_jump_step_off >= 0)
+ {
+ as = &state->steps[result_coercion_jump_step_off];
+ as->d.jump.jumpdone = coercion_finish_step_off;
+ }
+
+ /* The rest should jump to the end. */
+ foreach(lc, adjust_jumps)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+
+ /*
+ * Set RETURNING type's input function used by ExecEvalJsonExprCoercion().
+ */
+ if (jexpr->omit_quotes ||
+ (jexpr->result_coercion && jexpr->result_coercion->via_io))
+ {
+ Oid typinput;
+ FmgrInfo *finfo;
+
+ /* lookup the result type's input function */
+ getTypeInputInfo(jexpr->returning->typid, &typinput,
+ &jsestate->input.typioparam);
+ finfo = palloc0(sizeof(FmgrInfo));
+ fmgr_info(typinput, finfo);
+ jsestate->input.finfo = finfo;
+ }
+}
+
+/*
+ * Returns constant values to be returned to the user for various
+ * non-ERROR ON ERROR/EMPTY behaviors.
+ *
+ * Note that JSON_BEHAVIOR_DEFAULT expression is handled by the
+ * caller separately.
+ */
+static Datum
+GetJsonBehaviorConstVal(JsonBehavior *behavior, bool *is_null)
+{
+ *is_null = false;
+
+ switch (behavior->btype)
+ {
+ case JSON_BEHAVIOR_EMPTY_ARRAY:
+ return JsonbPGetDatum(JsonbMakeEmptyArray());
+
+ case JSON_BEHAVIOR_EMPTY_OBJECT:
+ return JsonbPGetDatum(JsonbMakeEmptyObject());
+
+ case JSON_BEHAVIOR_TRUE:
+ return BoolGetDatum(true);
+
+ case JSON_BEHAVIOR_FALSE:
+ return BoolGetDatum(false);
+
+ case JSON_BEHAVIOR_NULL:
+ case JSON_BEHAVIOR_UNKNOWN:
+ case JSON_BEHAVIOR_EMPTY:
+ *is_null = true;
+ return (Datum) 0;
+
+ case JSON_BEHAVIOR_DEFAULT:
+ /* Always handled in the caller. */
+ Assert(false);
+ return (Datum) 0;
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON behavior %d", behavior->btype);
+ return (Datum) 0;
+ }
+}
+
+/* Initialize one JsonCoercion for execution. */
+static JsonCoercionState *
+ExecInitJsonCoercion(ExprEvalStep *scratch, ExprState *state,
+ JsonCoercion *coercion, JsonBehavior *on_error,
+ Datum *resv, bool *resnull)
+{
+ JsonCoercionState *jcstate = palloc0(sizeof(JsonCoercionState));
+
+ jcstate->coercion = coercion;
+ if (coercion && coercion->expr)
+ {
+ Datum *save_innermost_caseval;
+ bool *save_innermost_casenull;
+ ErrorSaveContext *save_escontext;
+
+ jcstate->jump_eval_expr = state->steps_len;
+
+ /* Push step(s) to compute cstate->coercion. */
+ save_innermost_caseval = state->innermost_caseval;
+ save_innermost_casenull = state->innermost_casenull;
+ save_escontext = state->escontext;
+
+ state->innermost_caseval = resv;
+ state->innermost_casenull = resnull;
+ if (on_error->btype != JSON_BEHAVIOR_ERROR)
+ {
+ jcstate->escontext.type = T_ErrorSaveContext;
+ state->escontext = &jcstate->escontext;
+ }
+ else
+ state->escontext = NULL;
+
+ ExecInitExprRec((Expr *) coercion->expr, state, resv, resnull);
+
+ state->innermost_caseval = save_innermost_caseval;
+ state->innermost_casenull = save_innermost_casenull;
+ state->escontext = save_escontext;
+ }
+ else
+ jcstate->jump_eval_expr = -1;
+
+ return jcstate;
+}
+
+/*
+ * Push steps to evaluate coercions from a JSON_VALUE items specified in
+ * 'item_coercions'
+ */
+static List *
+ExecInitJsonItemCoercions(ExprEvalStep *scratch, ExprState *state,
+ List *item_coercions, JsonBehavior *on_error,
+ Datum *resv, bool *resnull)
+{
+ List *item_jcstates = NIL;
+ ExprEvalStep *as;
+ List *adjust_jumps = NIL;
+ ListCell *lc;
+
+ /* Push the steps of individual coercions. */
+ foreach(lc, item_coercions)
+ {
+ JsonCoercion *coercion = lfirst(lc);
+ JsonCoercionState *item_jcstate;
+
+ item_jcstate = ExecInitJsonCoercion(scratch, state, coercion,
+ on_error, resv, resnull);
+ item_jcstates = lappend(item_jcstates, item_jcstate);
+
+
+ /* Emit JUMP step to skip past other coercions' steps. */
+ scratch->opcode = EEOP_JUMP;
+
+ /*
+ * Remember JUMP step address to set the actual jump target address
+ * below.
+ */
+ adjust_jumps = lappend_int(adjust_jumps, state->steps_len);
+ ExprEvalPushStep(state, scratch);
+ }
+
+ foreach(lc, adjust_jumps)
+ {
+ int jump_step_id = lfirst_int(lc);
+
+ as = &state->steps[jump_step_id];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+
+ return item_jcstates;
+}
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index c8018da19f..9adf31682c 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -75,6 +75,7 @@
#include "utils/json.h"
#include "utils/jsonb.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/timestamp.h"
@@ -153,6 +154,9 @@ static TupleDesc get_cached_rowtype(Oid type_id, int32 typmod,
bool *changed);
static void ExecEvalRowNullInt(ExprState *state, ExprEvalStep *op,
ExprContext *econtext, bool checkisnull);
+static Datum ExecPrepareJsonItemCoercion(JsonbValue *item,
+ List *item_jcstates,
+ JsonCoercionState **p_item_jcstate);
/* fast-path evaluation functions */
static Datum ExecJustInnerVar(ExprState *state, ExprContext *econtext, bool *isnull);
@@ -481,6 +485,11 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_XMLEXPR,
&&CASE_EEOP_JSON_CONSTRUCTOR,
&&CASE_EEOP_IS_JSON,
+ &&CASE_EEOP_JSONEXPR_SKIP,
+ &&CASE_EEOP_JSONEXPR_PATH,
+ &&CASE_EEOP_JSONEXPR_BEHAVIOR,
+ &&CASE_EEOP_JSONEXPR_COERCION,
+ &&CASE_EEOP_JSONEXPR_COERCION_FINISH,
&&CASE_EEOP_AGGREF,
&&CASE_EEOP_GROUPING_FUNC,
&&CASE_EEOP_WINDOW_FUNC,
@@ -1192,7 +1201,8 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
* Should get null result if and only if str is NULL or if we
* got an error above.
*/
- if (str == NULL || SOFT_ERROR_OCCURRED(state->escontext))
+ if (str == NULL ||
+ SOFT_ERROR_OCCURRED(op->d.iocoerce.escontext))
Assert(*op->resnull);
else
Assert(!*op->resnull);
@@ -1539,6 +1549,38 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_JSONEXPR_PATH)
+ {
+ /* too complex for an inline implementation */
+ ExecEvalJsonExpr(state, op, econtext);
+ EEO_NEXT();
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_SKIP)
+ {
+ /* too complex for an inline implementation */
+ EEO_JUMP(ExecEvalJsonExprSkip(state, op));
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_BEHAVIOR)
+ {
+ /* too complex for an inline implementation */
+ EEO_JUMP(ExecEvalJsonExprBehavior(state, op));
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION)
+ {
+ /* too complex for an inline implementation */
+ EEO_JUMP(ExecEvalJsonExprCoercion(state, op, econtext,
+ *op->resvalue, *op->resnull));
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION_FINISH)
+ {
+ /* too complex for an inline implementation */
+ EEO_JUMP(ExecEvalJsonExprCoercionFinish(state, op));
+ }
+
EEO_CASE(EEOP_AGGREF)
{
/*
@@ -4134,6 +4176,507 @@ ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op)
*op->resvalue = BoolGetDatum(res);
}
+/*
+ * Evaluate given JsonExpr by performing the specified JSON operation.
+ *
+ * This also populates the JsonExprPostEvalState with the information needed
+ * by the subsequent steps that handle the specified JsonBehavior.
+ */
+void
+ExecEvalJsonExpr(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ JsonExprPreEvalState *pre_eval = &jsestate->pre_eval;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonExpr *jexpr = jsestate->jsexpr;
+ Datum item;
+ Datum res = (Datum) 0;
+ bool resnull = true;
+ JsonPath *path;
+ bool throw_error = (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
+ bool *error = &post_eval->error;
+ bool *empty = &post_eval->empty;
+
+ item = pre_eval->formatted_expr.value;
+ path = DatumGetJsonPathP(pre_eval->pathspec.value);
+
+ /* Reset JsonExprPostEvalState for this evaluation. */
+ memset(post_eval, 0, sizeof(*post_eval));
+
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ {
+ bool exists = JsonPathExists(item, path,
+ !throw_error ? error : NULL,
+ pre_eval->args);
+
+ post_eval->jcstate = jsestate->result_jcstate;
+ if (*error)
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ resnull = false;
+ res = BoolGetDatum(exists);
+ break;
+ }
+
+ case JSON_QUERY_OP:
+ res = JsonPathQuery(item, path, jexpr->wrapper, empty,
+ !throw_error ? error : NULL,
+ pre_eval->args);
+
+ post_eval->jcstate = jsestate->result_jcstate;
+ if (*error)
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+ resnull = !DatumGetPointer(res);
+ break;
+
+ case JSON_VALUE_OP:
+ {
+ JsonbValue *jbv = JsonPathValue(item, path, empty,
+ !throw_error ? error : NULL,
+ pre_eval->args);
+
+ /* Might get overridden below by an item_jcstate. */
+ post_eval->jcstate = jsestate->result_jcstate;
+ if (*error)
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ if (!jbv) /* NULL or empty */
+ {
+ resnull = true;
+ break;
+ }
+
+ Assert(!*empty);
+
+ resnull = false;
+
+ /* Coerce scalar item to the output type */
+
+ /*
+ * If the requested output type is json(b), use
+ * JsonExprState.result_coercion to do the coercion.
+ */
+ if (jexpr->returning->typid == JSONOID ||
+ jexpr->returning->typid == JSONBOID)
+ {
+ /* Use result_coercion from json[b] to the output type */
+ res = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ break;
+ }
+
+ /*
+ * Else, use one of the item_coercions.
+ *
+ * Error out if no cast exists to coerce SQL/JSON item to the
+ * the output type.
+ */
+ res = ExecPrepareJsonItemCoercion(jbv,
+ jsestate->item_jcstates,
+ &post_eval->jcstate);
+ if (post_eval->jcstate &&
+ post_eval->jcstate->coercion &&
+ (post_eval->jcstate->coercion->via_io ||
+ post_eval->jcstate->coercion->via_populate))
+ {
+ if (!throw_error)
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ /*
+ * Coercion via I/O means here that the cast to the target
+ * type simply does not exist.
+ */
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_ITEM_CANNOT_BE_CAST_TO_TARGET_TYPE),
+ errmsg("SQL/JSON item cannot be cast to target type")));
+ }
+ break;
+ }
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ /*
+ * If the ON EMPTY behavior is to cause an error, do so here. Other
+ * behaviors will be handled in ExecEvalJsonExprBehavior().
+ */
+ if (*empty)
+ {
+ Assert(jexpr->on_empty); /* it is not JSON_EXISTS */
+
+ if (jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
+ {
+ if (!throw_error)
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_NO_SQL_JSON_ITEM),
+ errmsg("no SQL/JSON item")));
+ }
+ }
+
+ *op->resvalue = res;
+ *op->resnull = resnull;
+}
+
+/*
+ * Skip calling ExecEvalJson() on the given JsonExpr?
+ *
+ * Returns the step address to be performed next.
+ */
+int
+ExecEvalJsonExprSkip(ExprState *state, ExprEvalStep *op)
+{
+ JsonExprState *jsestate = op->d.jsonexpr_skip.jsestate;
+
+ /*
+ * Skip if either of the input expressions has turned out to be NULL,
+ * though do execute domain checks for NULLs, which are handled by the
+ * coercion step.
+ */
+ if (jsestate->pre_eval.formatted_expr.isnull ||
+ jsestate->pre_eval.pathspec.isnull)
+ {
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+
+ /* ExecEvalJsonExprCoercion() depends on this. */
+ jsestate->post_eval.jcstate = jsestate->result_jcstate;
+
+ return op->d.jsonexpr_skip.jump_coercion;
+ }
+
+ /*
+ * Go evaluate the PASSING args if any and subsequently JSON path itself.
+ */
+ return op->d.jsonexpr_skip.jump_passing_args;
+}
+
+/*
+ * Returns the step address to perform the JsonBehavior applicable to
+ * the JSON item that resulted from evaluating the given JsonExpr.
+ *
+ * Returns the step address to be performed next.
+ */
+int
+ExecEvalJsonExprBehavior(ExprState *state, ExprEvalStep *op)
+{
+ JsonExprState *jsestate = op->d.jsonexpr_behavior.jsestate;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonBehavior *behavior = NULL;
+ int jump_to = -1;
+
+ if (post_eval->error || post_eval->coercion_error)
+ {
+ behavior = jsestate->jsexpr->on_error;
+ jump_to = op->d.jsonexpr_behavior.jump_onerror_expr;
+ }
+ else if (post_eval->empty)
+ {
+ behavior = jsestate->jsexpr->on_empty;
+ jump_to = op->d.jsonexpr_behavior.jump_onempty_expr;
+ }
+ else if (!post_eval->coercion_done)
+ {
+ /*
+ * If no error or the JSON item is not empty, directly go to the
+ * coercion step to coerce the item as is.
+ */
+ return op->d.jsonexpr_behavior.jump_coercion;
+ }
+
+ Assert(behavior);
+
+ /*
+ * Set up for coercion step that will run to coerce a non-default behavior
+ * value. It should use result_coercion, if any. Errors that may occur
+ * should be thrown for JSON ops other than JSON_VALUE_OP.
+ */
+ if (behavior->btype != JSON_BEHAVIOR_DEFAULT)
+ {
+ post_eval->jcstate = jsestate->result_jcstate;
+ post_eval->coercing_behavior_expr = true;
+ }
+
+ Assert(jump_to >= 0);
+ return jump_to;
+}
+
+/*
+ * Evaluate or return the step address to evaluate a coercion of a JSON item
+ * to the target type. The former if the coercion must be done right away by
+ * calling the target type's input function, and for some types, by calling
+ * json_populate_type().
+ *
+ * Returns the step address to be performed next.
+ */
+int
+ExecEvalJsonExprCoercion(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext,
+ Datum res, bool resnull)
+{
+ JsonExprState *jsestate = op->d.jsonexpr_coercion.jsestate;
+ JsonExpr *jexpr = jsestate->jsexpr;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonCoercionState *jcstate = post_eval->jcstate;
+ char *val_string = NULL;
+ bool omit_quotes = false;
+
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ if (jcstate && jcstate->jump_eval_expr >= 0)
+ return jcstate->jump_eval_expr;
+
+ /* No coercion needed. */
+ post_eval->coercion_done = true;
+ return op->d.jsonexpr_coercion.jump_coercion_done;
+
+ case JSON_QUERY_OP:
+ if (jexpr->omit_quotes)
+ {
+ Jsonb *jb = resnull ? NULL : DatumGetJsonbP(res);
+
+ if (jb && JB_ROOT_IS_SCALAR(jb))
+ {
+ omit_quotes = true;
+ val_string = JsonbUnquote(jb);
+ }
+ }
+ else if (jcstate && jcstate->jump_eval_expr >= 0)
+ return jcstate->jump_eval_expr;
+ break;
+
+ case JSON_VALUE_OP:
+ if (jcstate != jsestate->result_jcstate)
+ {
+ if (jcstate->jump_eval_expr >= 0)
+ return jcstate->jump_eval_expr;
+
+ /* No coercion needed. */
+ post_eval->coercion_done = true;
+ return op->d.jsonexpr_coercion.jump_coercion_done;
+ }
+ else if (jcstate && jcstate->jump_eval_expr >= 0)
+ return jcstate->jump_eval_expr;
+ break;
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
+ break;
+ }
+
+ /*
+ * OK, there's no coercion expression, so coerce either by directly
+ * calling the input function or by calling json_populate_type().
+ */
+ if (jcstate || omit_quotes)
+ {
+ ErrorSaveContext escontext = {T_ErrorSaveContext};
+ Node *escontext_p = NULL;
+ JsonCoercion *coercion = jcstate ? jcstate->coercion : NULL;
+ bool type_is_domain =
+ (getBaseType(jexpr->returning->typid) != jexpr->returning->typid);
+
+ /*
+ * For JSON_QUERY_OP, throw the errors that occur when coercing a
+ * non-default JsonBehavior expression. Also throw an error if
+ * coercing via_io and the returning type is a domain, whose
+ * constraint violations must be reported.
+ *
+ * In all other cases, respect the ON ERROR clause.
+ */
+ if ((jexpr->op == JSON_QUERY_OP &&
+ post_eval->coercing_behavior_expr) ||
+ (coercion && coercion->via_io && type_is_domain))
+ escontext_p = NULL;
+ else if (jexpr->on_error->btype != JSON_BEHAVIOR_ERROR)
+ escontext_p = (Node *) &escontext;
+
+ if ((coercion && coercion->via_io) || omit_quotes)
+ {
+ if (!InputFunctionCallSafe(jsestate->input.finfo, val_string,
+ jsestate->input.typioparam,
+ jexpr->returning->typmod,
+ escontext_p,
+ op->resvalue))
+ {
+ post_eval->coercion_error = true;
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return op->d.jsonexpr_coercion.jump_coercion_error;
+ }
+ }
+ else if (coercion && coercion->via_populate)
+ {
+ *op->resvalue = json_populate_type(res, JSONBOID,
+ jexpr->returning->typid,
+ jexpr->returning->typmod,
+ &post_eval->cache,
+ econtext->ecxt_per_query_memory,
+ op->resnull,
+ escontext_p);
+ if (SOFT_ERROR_OCCURRED(escontext_p))
+ {
+ post_eval->coercion_error = true;
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ return op->d.jsonexpr_coercion.jump_coercion_error;
+ }
+ }
+ }
+
+ post_eval->coercion_done = true;
+ return op->d.jsonexpr_coercion.jump_coercion_done;
+}
+
+/*
+ * Checks if the coercion evaluation led to an error. If an error did occur,
+ * this returns the address of the step that handles the error, otherwise
+ * the step after the coercion step, which finishes the JsonExpr processing.
+ */
+int
+ExecEvalJsonExprCoercionFinish(ExprState *state, ExprEvalStep *op)
+{
+ JsonExprPostEvalState *post_eval =
+ &op->d.jsonexpr_coercion_finish.jsestate->post_eval;
+
+ if (SOFT_ERROR_OCCURRED(&post_eval->jcstate->escontext))
+ {
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ post_eval->coercion_error = true;
+
+ /*
+ * Also make ErrorSaveContext ready for the next row. Since we never
+ * set details_wanted, we don't need to also reset error_data, which
+ * would be NULL anyway.
+ */
+ Assert(!post_eval->jcstate->escontext.details_wanted &&
+ post_eval->jcstate->escontext.error_data == NULL);
+ post_eval->jcstate->escontext.error_occurred = false;
+
+ return op->d.jsonexpr_coercion_finish.jump_coercion_error;
+ }
+
+ post_eval->coercion_done = true;
+ return op->d.jsonexpr_coercion_finish.jump_coercion_done;
+}
+
+/*
+ * Prepare SQL/JSON item coercion to the output type. Returned a datum of the
+ * corresponding SQL type and a pointer to the coercion state.
+ */
+static Datum
+ExecPrepareJsonItemCoercion(JsonbValue *item, List *item_jcstates,
+ JsonCoercionState **p_item_jcstate)
+{
+ JsonCoercionState *item_jcstate;
+ Datum res;
+ JsonbValue buf;
+
+ if (item->type == jbvBinary &&
+ JsonContainerIsScalar(item->val.binary.data))
+ {
+ bool res PG_USED_FOR_ASSERTS_ONLY;
+
+ res = JsonbExtractScalar(item->val.binary.data, &buf);
+ item = &buf;
+ Assert(res);
+ }
+
+ /* get coercion state reference and datum of the corresponding SQL type */
+ switch (item->type)
+ {
+ case jbvNull:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeNull);
+ res = (Datum) 0;
+ break;
+
+ case jbvString:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeString);
+ res = PointerGetDatum(cstring_to_text_with_len(item->val.string.val,
+ item->val.string.len));
+ break;
+
+ case jbvNumeric:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeNumeric);
+ res = NumericGetDatum(item->val.numeric);
+ break;
+
+ case jbvBool:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeBoolean);
+ res = BoolGetDatum(item->val.boolean);
+ break;
+
+ case jbvDatetime:
+ res = item->val.datetime.value;
+ switch (item->val.datetime.typid)
+ {
+ case DATEOID:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeDate);
+ break;
+ case TIMEOID:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeTime);
+ break;
+ case TIMETZOID:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeTimetz);
+ break;
+ case TIMESTAMPOID:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeTimestamp);
+ break;
+ case TIMESTAMPTZOID:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeTimestamptz);
+ break;
+ default:
+ elog(ERROR, "unexpected jsonb datetime type oid %u",
+ item->val.datetime.typid);
+ return (Datum) 0;
+ }
+ break;
+
+ case jbvArray:
+ case jbvObject:
+ case jbvBinary:
+ item_jcstate = list_nth(item_jcstates, JsonItemTypeComposite);
+ res = JsonbPGetDatum(JsonbValueToJsonb(item));
+ break;
+
+ default:
+ elog(ERROR, "unexpected jsonb value type %d", item->type);
+ return (Datum) 0;
+ }
+
+ *p_item_jcstate = item_jcstate;
+
+ return res;
+}
+
/*
* ExecEvalGroupingFunc
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index 7d44a4c9f4..0e06af15b4 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1875,6 +1875,271 @@ llvm_compile_expr(ExprState *state)
LLVMBuildBr(b, opblocks[opno + 1]);
break;
+ case EEOP_JSONEXPR_PATH:
+ build_EvalXFunc(b, mod, "ExecEvalJsonExpr",
+ v_state, op, v_econtext);
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
+ case EEOP_JSONEXPR_SKIP:
+ {
+ LLVMValueRef params[2];
+ LLVMValueRef v_ret;
+
+ /*
+ * Call ExecEvalJsonExprSkip() to decide if JSON path
+ * evaluation can be skipped. This returns the step
+ * address to jump to.
+ */
+ params[0] = v_state;
+ params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
+ v_ret = LLVMBuildCall(b,
+ llvm_pg_func(mod, "ExecEvalJsonExprSkip"),
+ params, lengthof(params), "");
+
+ /*
+ * Jump to coercion step if the returned address is the
+ * same as jsonexpr_skip.jump_coercion, which signifies
+ * skipping of JSON path evaluation, else to the next step
+ * which must point to the steps to evaluate PASSING args,
+ * if any, or to the JSON path evaluation.
+ */
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_int32_const(lc, op->d.jsonexpr_skip.jump_coercion),
+ ""),
+ opblocks[op->d.jsonexpr_skip.jump_coercion],
+ opblocks[opno + 1]);
+ break;
+ }
+
+ case EEOP_JSONEXPR_BEHAVIOR:
+ {
+ LLVMValueRef params[2];
+ LLVMValueRef v_ret;
+ LLVMBasicBlockRef b_jump_onerror_default;
+
+ /*
+ * Call ExecEvalJsonExprBehavior() to decide if ON EMPTY
+ * or ON ERROR behavior must be invoked depending on what
+ * JSON path evaluation returned. This returns the step
+ * address to jump to.
+ */
+ params[0] = v_state;
+ params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
+ v_ret = LLVMBuildCall(b,
+ llvm_pg_func(mod, "ExecEvalJsonExprBehavior"),
+ params, lengthof(params), "");
+
+ b_jump_onerror_default =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_behavior_jump_onerror_default", opno);
+
+ /*
+ * Jump to coercion step if the returned address is the
+ * same as jsonexpr_behavior.jump_coercion, else to the
+ * next block, one that checks whether to evaluate the ON
+ * ERROR default expression.
+ */
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_int32_const(lc, op->d.jsonexpr_behavior.jump_coercion),
+ ""),
+ opblocks[op->d.jsonexpr_behavior.jump_coercion],
+ b_jump_onerror_default);
+
+ /*
+ * Block that checks whether to evaluate the ON ERROR
+ * default expression.
+ *
+ * Jump to evaluate the ON ERROR default expression if the
+ * returned address is the same as
+ * jsonexpr_behavior.jump_onerror_default, else jump to
+ * evaluate the ON EMPTY default expression.
+ */
+ LLVMPositionBuilderAtEnd(b, b_jump_onerror_default);
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_int32_const(lc, op->d.jsonexpr_behavior.jump_onerror_expr),
+ ""),
+ opblocks[op->d.jsonexpr_behavior.jump_onerror_expr],
+ opblocks[op->d.jsonexpr_behavior.jump_onempty_expr]);
+ break;
+ }
+ case EEOP_JSONEXPR_COERCION:
+ {
+ JsonExprState *jsestate = op->d.jsonexpr_coercion.jsestate;
+ List *item_jcstates = jsestate->item_jcstates;
+ JsonCoercionState *result_jcstate = jsestate->result_jcstate;
+ LLVMValueRef v_ret;
+ LLVMValueRef params[5];
+ LLVMBasicBlockRef b_jump_result_jcstate;
+ LLVMBasicBlockRef b_jump_item_jcstates;
+
+ /*
+ * Call ExecEvalJsonExprCoercion() to evaluate appropriate
+ * coercion. This will return the step address to jump
+ * to.
+ */
+ params[0] = v_state;
+ params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
+ params[2] = v_econtext;
+ params[3] = LLVMBuildLoad(b, v_resvaluep, "");
+ params[4] = LLVMBuildLoad(b, v_resnullp, "");
+ v_ret = LLVMBuildCall(b,
+ llvm_pg_func(mod, "ExecEvalJsonExprCoercion"),
+ params, lengthof(params), "");
+
+ b_jump_result_jcstate =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_coercion_result_jcstate", opno);
+ b_jump_item_jcstates =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_coercion_item_jcstates", opno);
+
+ /*
+ * Jump to handle a coercion error if the returned address
+ * is the same as jsonexpr_coercion.jump_coercion_error,
+ * else to the step after coercion (coercion done!).
+ */
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_int32_const(lc, op->d.jsonexpr_coercion.jump_coercion_error),
+ ""),
+ opblocks[op->d.jsonexpr_coercion.jump_coercion_error],
+ b_jump_result_jcstate);
+
+ /*
+ * Jump to evaluate the result_coercion's expression if
+ * there's one.
+ */
+ LLVMPositionBuilderAtEnd(b, b_jump_result_jcstate);
+ if (result_jcstate)
+ {
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_int32_const(lc, result_jcstate->jump_eval_expr),
+ ""),
+ result_jcstate->jump_eval_expr >= 0 ?
+ opblocks[result_jcstate->jump_eval_expr] :
+ opblocks[op->d.jsonexpr_coercion.jump_coercion_done],
+ b_jump_item_jcstates);
+ }
+ else
+ LLVMBuildBr(b, b_jump_item_jcstates);
+
+ LLVMPositionBuilderAtEnd(b, b_jump_item_jcstates);
+ if (item_jcstates)
+ {
+ int n_coercions = list_length(item_jcstates);
+ ListCell *l;
+ int i;
+ LLVMBasicBlockRef *b_coercions;
+
+
+ /*
+ * Will create a block for each coercion below to
+ * check whether to evaluate the coercion's expression
+ * if there's one or to skip to the end if not.
+ */
+ b_coercions = palloc((n_coercions + 1) * sizeof(LLVMBasicBlockRef));
+ for (i = 0; i < n_coercions + 1; i++)
+ b_coercions[i] =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.json_item_coercion.%d",
+ opno, i);
+
+ /* Jump to check first coercion */
+ LLVMBuildBr(b, b_coercions[0]);
+
+ /*
+ * Add conditional branches for individual coercion's
+ * expressions
+ */
+ i = 0;
+ foreach(l, item_jcstates)
+ {
+ JsonCoercionState *item_jcstate = lfirst(l);
+
+ /* Block for this coercion */
+ LLVMPositionBuilderAtEnd(b, b_coercions[i]);
+
+ /*
+ * Jump to evaluate the coercion's expression if
+ * the address returned is the same as this
+ * coercion's jump_eval_expr (that is, if it is
+ * valid), else check the next coercion's.
+ */
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_int32_const(lc, item_jcstate->jump_eval_expr),
+ ""),
+ item_jcstate->jump_eval_expr >= 0 ?
+ opblocks[item_jcstate->jump_eval_expr] :
+ opblocks[op->d.jsonexpr_coercion.jump_coercion_done],
+ b_coercions[i + 1]);
+ i++;
+ }
+
+ /*
+ * A placeholder block that the last coercion's block
+ * might jump to, which unconditionally jumps to end
+ * of coercions.
+ */
+ LLVMPositionBuilderAtEnd(b, b_coercions[i]);
+ LLVMBuildBr(b, opblocks[op->d.jsonexpr_coercion.jump_coercion_done]);
+ }
+ else
+ LLVMBuildBr(b, opblocks[op->d.jsonexpr_coercion.jump_coercion_done]);
+ break;
+ }
+
+ case EEOP_JSONEXPR_COERCION_FINISH:
+ {
+ LLVMValueRef params[2];
+ LLVMValueRef v_ret;
+
+ /*
+ * Call ExecEvalJsonExprCoercionFinish() to check whether
+ * an coercion error occurred, in which case we must jump
+ * to whatever step handles the error. This returns the
+ * step address to jump to.
+ */
+ params[0] = v_state;
+ params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
+ v_ret = LLVMBuildCall(b,
+ llvm_pg_func(mod, "ExecEvalJsonExprCoercionFinish"),
+ params, lengthof(params), "");
+
+ /*
+ * Jump to the step that handles coercion error if the
+ * returned address is the same as
+ * jsonexpr_coercion_finish.jump_coercion_error, else to
+ * jsonexpr_coercion_finish.jump_coercion_done.
+ */
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_int32_const(lc, op->d.jsonexpr_coercion_finish.jump_coercion_error),
+ ""),
+ opblocks[op->d.jsonexpr_coercion_finish.jump_coercion_error],
+ opblocks[op->d.jsonexpr_coercion_finish.jump_coercion_done]);
+ break;
+ }
+
case EEOP_AGGREF:
{
LLVMValueRef v_aggno;
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index e1e9625038..3986b00341 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -137,6 +137,11 @@ void *referenced_functions[] =
ExecEvalXmlExpr,
ExecEvalJsonConstructor,
ExecEvalJsonIsPredicate,
+ ExecEvalJsonExprSkip,
+ ExecEvalJsonExprBehavior,
+ ExecEvalJsonExprCoercion,
+ ExecEvalJsonExprCoercionFinish,
+ ExecEvalJsonExpr,
MakeExpandedObjectReadOnlyInternal,
InputFunctionCallSafe,
slot_getmissingattrs,
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index 0e7e6e46d9..e1f7fde2bd 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -860,6 +860,22 @@ makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
return jve;
}
+/*
+ * makeJsonBehavior -
+ * creates a JsonBehavior node
+ */
+JsonBehavior *
+makeJsonBehavior(JsonBehaviorType type, Node *default_expr, int location)
+{
+ JsonBehavior *behavior = makeNode(JsonBehavior);
+
+ behavior->btype = type;
+ behavior->default_expr = default_expr;
+ behavior->location = location;
+
+ return behavior;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index c03f4f23e2..ef08ef2cbe 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -234,6 +234,12 @@ exprType(const Node *expr)
case T_JsonIsPredicate:
type = BOOLOID;
break;
+ case T_JsonExpr:
+ type = ((const JsonExpr *) expr)->returning->typid;
+ break;
+ case T_JsonCoercion:
+ type = exprType(((const JsonCoercion *) expr)->expr);
+ break;
case T_NullTest:
type = BOOLOID;
break;
@@ -493,6 +499,10 @@ exprTypmod(const Node *expr)
return exprTypmod((Node *) ((const JsonValueExpr *) expr)->formatted_expr);
case T_JsonConstructorExpr:
return ((const JsonConstructorExpr *) expr)->returning->typmod;
+ case T_JsonExpr:
+ return ((JsonExpr *) expr)->returning->typmod;
+ case T_JsonCoercion:
+ return exprTypmod(((const JsonCoercion *) expr)->expr);
case T_CoerceToDomain:
return ((const CoerceToDomain *) expr)->resulttypmod;
case T_CoerceToDomainValue:
@@ -969,6 +979,22 @@ exprCollation(const Node *expr)
/* IS JSON's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) expr;
+ JsonCoercion *coercion = jexpr->result_coercion;
+
+ if (!coercion)
+ coll = InvalidOid;
+ else if (coercion->expr)
+ coll = exprCollation(coercion->expr);
+ else if (coercion->via_io || coercion->via_populate)
+ coll = coercion->collation;
+ else
+ coll = InvalidOid;
+ }
+ break;
+
case T_NullTest:
/* NullTest's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
@@ -1160,6 +1186,9 @@ exprSetCollation(Node *expr, Oid collation)
case T_CaseExpr:
((CaseExpr *) expr)->casecollid = collation;
break;
+ case T_CaseTestExpr:
+ ((CaseTestExpr *) expr)->collation = collation;
+ break;
case T_ArrayExpr:
((ArrayExpr *) expr)->array_collid = collation;
break;
@@ -1205,6 +1234,29 @@ exprSetCollation(Node *expr, Oid collation)
case T_JsonIsPredicate:
Assert(!OidIsValid(collation)); /* result is always boolean */
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) expr;
+
+ if (jexpr->result_coercion)
+ exprSetCollation((Node *) jexpr->result_coercion, collation);
+ else
+ Assert(!OidIsValid(collation)); /* result is always a
+ * json[b] type */
+ }
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) expr;
+
+ if (coercion->expr)
+ exprSetCollation(coercion->expr, collation);
+ else if (coercion->via_io || coercion->via_populate)
+ coercion->collation = collation;
+ else
+ Assert(!OidIsValid(collation));
+ }
+ break;
case T_NullTest:
/* NullTest's result is boolean ... */
Assert(!OidIsValid(collation)); /* ... so never set a collation */
@@ -1508,6 +1560,15 @@ exprLocation(const Node *expr)
case T_JsonIsPredicate:
loc = ((const JsonIsPredicate *) expr)->location;
break;
+ case T_JsonExpr:
+ {
+ const JsonExpr *jsexpr = (const JsonExpr *) expr;
+
+ /* consider both function name and leftmost arg */
+ loc = leftmostLoc(jsexpr->location,
+ exprLocation(jsexpr->formatted_expr));
+ }
+ break;
case T_NullTest:
{
const NullTest *nexpr = (const NullTest *) expr;
@@ -2260,6 +2321,28 @@ expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ if (WALK(jexpr->formatted_expr))
+ return true;
+ if (WALK(jexpr->result_coercion))
+ return true;
+ if (WALK(jexpr->item_coercions))
+ return true;
+ if (WALK(jexpr->passing_values))
+ return true;
+ /* we assume walker doesn't care about passing_names */
+ if (jexpr->on_empty &&
+ WALK(jexpr->on_empty->default_expr))
+ return true;
+ if (WALK(jexpr->on_error->default_expr))
+ return true;
+ }
+ break;
+ case T_JsonCoercion:
+ return WALK(((JsonCoercion *) node)->expr);
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
@@ -3259,6 +3342,36 @@ expression_tree_mutator_impl(Node *node,
return (Node *) newnode;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+ JsonExpr *newnode;
+
+ FLATCOPY(newnode, jexpr, JsonExpr);
+ MUTATE(newnode->path_spec, jexpr->path_spec, Node *);
+ MUTATE(newnode->formatted_expr, jexpr->formatted_expr, Node *);
+ MUTATE(newnode->result_coercion, jexpr->result_coercion, JsonCoercion *);
+ MUTATE(newnode->item_coercions, jexpr->item_coercions, List *);
+ MUTATE(newnode->passing_values, jexpr->passing_values, List *);
+ /* assume mutator does not care about passing_names */
+ if (newnode->on_empty)
+ MUTATE(newnode->on_empty->default_expr,
+ jexpr->on_empty->default_expr, Node *);
+ MUTATE(newnode->on_error->default_expr,
+ jexpr->on_error->default_expr, Node *);
+ return (Node *) newnode;
+ }
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) node;
+ JsonCoercion *newnode;
+
+ FLATCOPY(newnode, coercion, JsonCoercion);
+ MUTATE(newnode->expr, coercion->expr, Node *);
+ return (Node *) newnode;
+ }
+ break;
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -3945,6 +4058,43 @@ raw_expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonArgument:
+ return WALK(((JsonArgument *) node)->val);
+ case T_JsonCommon:
+ {
+ JsonCommon *jc = (JsonCommon *) node;
+
+ if (WALK(jc->expr))
+ return true;
+ if (WALK(jc->pathspec))
+ return true;
+ if (WALK(jc->passing))
+ return true;
+ }
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *jb = (JsonBehavior *) node;
+
+ if (jb->btype == JSON_BEHAVIOR_DEFAULT &&
+ WALK(jb->default_expr))
+ return true;
+ }
+ break;
+ case T_JsonFuncExpr:
+ {
+ JsonFuncExpr *jfe = (JsonFuncExpr *) node;
+
+ if (WALK(jfe->common))
+ return true;
+ if (jfe->output && WALK(jfe->output))
+ return true;
+ if (WALK(jfe->on_empty))
+ return true;
+ if (WALK(jfe->on_error))
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index d6ceafd51c..3bb48b64d5 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -4850,7 +4850,8 @@ cost_qual_eval_walker(Node *node, cost_qual_eval_context *context)
IsA(node, SQLValueFunction) ||
IsA(node, XmlExpr) ||
IsA(node, CoerceToDomain) ||
- IsA(node, NextValueExpr))
+ IsA(node, NextValueExpr) ||
+ IsA(node, JsonExpr))
{
/* Treat all these as having cost 1 */
context->total.per_tuple += cpu_operator_cost;
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index da258968b8..e40cfab4b7 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -53,6 +53,7 @@
#include "utils/fmgroids.h"
#include "utils/json.h"
#include "utils/jsonb.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/syscache.h"
@@ -412,6 +413,24 @@ contain_mutable_functions_walker(Node *node, void *context)
/* Check all subnodes */
}
+ if (IsA(node, JsonExpr))
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+ Const *cnst;
+
+ if (!IsA(jexpr->path_spec, Const))
+ return true;
+
+ cnst = castNode(Const, jexpr->path_spec);
+
+ Assert(cnst->consttype == JSONPATHOID);
+ if (cnst->constisnull)
+ return false;
+
+ return jspIsMutable(DatumGetJsonPathP(cnst->constvalue),
+ jexpr->passing_names, jexpr->passing_values);
+ }
+
if (IsA(node, SQLValueFunction))
{
/* all variants of SQLValueFunction are stable */
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 7d2032885e..ffa8bbe770 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -278,6 +278,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ JsonBehavior *jsbehavior;
+ JsonQuotes js_quotes;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -650,14 +652,22 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_returning_clause_opt
json_name_and_value
json_aggregate_func
+ json_api_common_syntax
+ json_argument
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
+ json_arguments
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
+ json_wrapper_behavior
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <jsbehavior> json_value_behavior
+ json_query_behavior
+ json_exists_behavior
+%type <js_quotes> json_quotes_clause_opt
/*
@@ -695,7 +705,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ COMMITTED COMPRESSION CONCURRENTLY CONDITIONAL CONFIGURATION CONFLICT
CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
@@ -706,8 +716,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
- EACH ELSE ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ESCAPE EVENT EXCEPT
- EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
+ EACH ELSE EMPTY_P ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ERROR_P ESCAPE
+ EVENT EXCEPT EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
EXTENSION EXTERNAL EXTRACT
FALSE_P FAMILY FETCH FILTER FINALIZE FIRST_P FLOAT_P FOLLOWING FOR
@@ -722,10 +732,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
- JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_OBJECT JSON_OBJECTAGG
- JSON_SCALAR JSON_SERIALIZE
+ JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
- KEY KEYS
+ KEY KEYS KEEP
LABEL LANGUAGE LARGE_P LAST_P LATERAL_P
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
@@ -739,7 +749,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
- OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR
+ OBJECT_P OF OFF OFFSET OIDS OLD OMIT ON ONLY OPERATOR OPTION OPTIONS OR
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
@@ -748,7 +758,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
- QUOTE
+ QUOTE QUOTES
RANGE READ REAL REASSIGN RECHECK RECURSIVE REF_P REFERENCES REFERENCING
REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA
@@ -759,7 +769,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
SEQUENCE SEQUENCES
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
- START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
+ START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
@@ -767,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TREAT TRIGGER TRIM TRUE_P
TRUNCATE TRUSTED TYPE_P TYPES_P
- UESCAPE UNBOUNDED UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
+ UESCAPE UNBOUNDED UNCONDITIONAL UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
UNLISTEN UNLOGGED UNTIL UPDATE USER USING
VACUUM VALID VALIDATE VALIDATOR VALUE_P VALUES VARCHAR VARIADIC VARYING
@@ -15711,6 +15721,192 @@ func_expr_common_subexpr:
n->location = @1;
$$ = (Node *) n;
}
+ | JSON_QUERY '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_QUERY_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->wrapper = $5;
+ if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@6)));
+ n->quotes = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_QUERY '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_query_behavior ON EMPTY_P
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_QUERY_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->wrapper = $5;
+ if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@6)));
+ n->quotes = $6;
+ n->on_empty = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_QUERY '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_query_behavior ON ERROR_P
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_QUERY_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->wrapper = $5;
+ if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@6)));
+ n->quotes = $6;
+ n->on_error = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_QUERY '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_query_behavior ON EMPTY_P
+ json_query_behavior ON ERROR_P
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_QUERY_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->wrapper = $5;
+ if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@6)));
+ n->quotes = $6;
+ n->on_empty = $7;
+ n->on_error = $10;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_EXISTS '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ ')'
+ {
+ JsonFuncExpr *p = makeNode(JsonFuncExpr);
+
+ p->op = JSON_EXISTS_OP;
+ p->common = (JsonCommon *) $3;
+ p->output = (JsonOutput *) $4;
+ p->location = @1;
+ $$ = (Node *) p;
+ }
+ | JSON_EXISTS '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_exists_behavior ON ERROR_P
+ ')'
+ {
+ JsonFuncExpr *p = makeNode(JsonFuncExpr);
+
+ p->op = JSON_EXISTS_OP;
+ p->common = (JsonCommon *) $3;
+ p->output = (JsonOutput *) $4;
+ p->on_error = $5;
+ p->location = @1;
+ $$ = (Node *) p;
+ }
+ | JSON_VALUE '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_VALUE_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+
+ | JSON_VALUE '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_value_behavior ON EMPTY_P
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_VALUE_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->on_empty = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_VALUE '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_value_behavior ON ERROR_P
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_VALUE_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->on_error = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+
+ | JSON_VALUE '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_value_behavior ON EMPTY_P
+ json_value_behavior ON ERROR_P
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_VALUE_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->on_empty = $5;
+ n->on_error = $8;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
;
@@ -16437,6 +16633,72 @@ opt_asymmetric: ASYMMETRIC
;
/* SQL/JSON support */
+json_api_common_syntax:
+ json_value_expr ',' a_expr /* i.e. a json_path */
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->passing = NULL;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | json_value_expr ',' a_expr AS name
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->passing = NULL;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+
+ | json_value_expr ',' a_expr /* i.e. a json_path */
+ PASSING json_arguments
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->passing = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | json_value_expr ',' a_expr AS name
+ PASSING json_arguments
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->passing = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_arguments:
+ json_argument { $$ = list_make1($1); }
+ | json_arguments ',' json_argument { $$ = lappend($1, $3); }
+ ;
+
+json_argument:
+ json_value_expr AS ColLabel
+ {
+ JsonArgument *n = makeNode(JsonArgument);
+
+ n->val = (JsonValueExpr *) $1;
+ n->name = $3;
+ $$ = (Node *) n;
+ }
+ ;
+
json_value_expr:
a_expr json_format_clause_opt
{
@@ -16462,6 +16724,50 @@ json_encoding_clause_opt:
| /* EMPTY */ { $$ = JS_ENC_DEFAULT; }
;
+json_query_behavior:
+ ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
+ | DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
+ | EMPTY_P ARRAY { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P OBJECT_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
+ /* non-standard, for Oracle compatibility only */
+ | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ ;
+
+json_exists_behavior:
+ ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | TRUE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
+ | FALSE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
+ | UNKNOWN { $$ = makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
+ ;
+
+json_value_behavior:
+ NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
+ | ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
+ ;
+
+/* ARRAY is a noise word */
+json_wrapper_behavior:
+ WITHOUT WRAPPER { $$ = JSW_NONE; }
+ | WITHOUT ARRAY WRAPPER { $$ = JSW_NONE; }
+ | WITH WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL ARRAY WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | /* empty */ { $$ = JSW_NONE; }
+ ;
+
+json_quotes_clause_opt:
+ KEEP QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_KEEP; }
+ | KEEP QUOTES { $$ = JS_QUOTES_KEEP; }
+ | OMIT QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_OMIT; }
+ | OMIT QUOTES { $$ = JS_QUOTES_OMIT; }
+ | /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -17064,6 +17370,7 @@ unreserved_keyword:
| COMMIT
| COMMITTED
| COMPRESSION
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17100,10 +17407,12 @@ unreserved_keyword:
| DOUBLE_P
| DROP
| EACH
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17153,6 +17462,7 @@ unreserved_keyword:
| INSTEAD
| INVOKER
| ISOLATION
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17199,6 +17509,7 @@ unreserved_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| OPERATOR
| OPTION
| OPTIONS
@@ -17229,6 +17540,7 @@ unreserved_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REASSIGN
@@ -17288,6 +17600,7 @@ unreserved_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUPPORT
@@ -17310,6 +17623,7 @@ unreserved_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNKNOWN
| UNLISTEN
@@ -17370,10 +17684,13 @@ col_name_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
| LEAST
| NATIONAL
| NCHAR
@@ -17606,6 +17923,7 @@ bare_label_keyword:
| COMMITTED
| COMPRESSION
| CONCURRENTLY
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17658,11 +17976,13 @@ bare_label_keyword:
| DROP
| EACH
| ELSE
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| END_P
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17732,10 +18052,14 @@ bare_label_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17796,6 +18120,7 @@ bare_label_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| ONLY
| OPERATOR
| OPTION
@@ -17833,6 +18158,7 @@ bare_label_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REAL
@@ -17901,6 +18227,7 @@ bare_label_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUBSTRING
@@ -17935,6 +18262,7 @@ bare_label_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNIQUE
| UNKNOWN
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64c582c344..21979fd64f 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -90,6 +90,23 @@ static Node *transformJsonParseExpr(ParseState *pstate, JsonParseExpr *expr);
static Node *transformJsonScalarExpr(ParseState *pstate, JsonScalarExpr *expr);
static Node *transformJsonSerializeExpr(ParseState *pstate,
JsonSerializeExpr *expr);
+static Node *transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *p);
+static JsonExpr *transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName);
+static void transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names);
+static JsonCoercion *coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr);
+static Oid JsonFuncExprDefaultReturnType(JsonExpr *jsexpr);
+static JsonCoercion *coerceJsonExpr(ParseState *pstate, Node *expr,
+ const JsonReturning *returning);
+static List *InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId);
+static JsonBehavior *transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning);
+static JsonBehavior *coerceJsonBehaviorDefaultExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning);
static Node *make_row_comparison_op(ParseState *pstate, List *opname,
List *largs, List *rargs, int location);
static Node *make_row_distinct_op(ParseState *pstate, List *opname,
@@ -353,6 +370,10 @@ transformExprRecurse(ParseState *pstate, Node *expr)
result = transformJsonSerializeExpr(pstate, (JsonSerializeExpr *) expr);
break;
+ case T_JsonFuncExpr:
+ result = transformJsonFuncExpr(pstate, (JsonFuncExpr *) expr);
+ break;
+
default:
/* should not reach here */
elog(ERROR, "unrecognized node type: %d", (int) nodeTag(expr));
@@ -3229,7 +3250,7 @@ makeJsonByteaToTextConversion(Node *expr, JsonFormat *format, int location)
static Node *
transformJsonValueExpr(ParseState *pstate, const char *constructName,
JsonValueExpr *ve, JsonFormatType default_format,
- Oid targettype)
+ Oid targettype, bool isarg)
{
Node *expr = transformExprRecurse(pstate, (Node *) ve->raw_expr);
Node *rawexpr;
@@ -3261,6 +3282,35 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
else
format = ve->format->format_type;
}
+ else if (isarg)
+ {
+ /* Pass SQL/JSON item types directly without conversion to json[b]. */
+ switch (exprtype)
+ {
+ case TEXTOID:
+ case NUMERICOID:
+ case BOOLOID:
+ case INT2OID:
+ case INT4OID:
+ case INT8OID:
+ case FLOAT4OID:
+ case FLOAT8OID:
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ return expr;
+
+ default:
+ if (typcategory == TYPCATEGORY_STRING)
+ return expr;
+ /* else convert argument to json[b] type */
+ break;
+ }
+
+ format = default_format;
+ }
else if (exprtype == JSONOID || exprtype == JSONBOID)
format = JS_FORMAT_DEFAULT; /* do not format json[b] types */
else
@@ -3272,7 +3322,8 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
Node *coerced;
bool only_allow_cast = OidIsValid(targettype);
- if (!only_allow_cast &&
+ if (!isarg &&
+ !only_allow_cast &&
exprtype != BYTEAOID && typcategory != TYPCATEGORY_STRING)
ereport(ERROR,
errcode(ERRCODE_DATATYPE_MISMATCH),
@@ -3425,6 +3476,11 @@ transformJsonOutput(ParseState *pstate, const JsonOutput *output,
errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("returning SETOF types is not supported in SQL/JSON functions"));
+ if (get_typtype(ret->typid) == TYPTYPE_PSEUDO)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("returning pseudo-types is not supported in SQL/JSON functions"));
+
if (ret->format->format_type == JS_FORMAT_DEFAULT)
/* assign JSONB format when returning jsonb, or JSON format otherwise */
ret->format->format_type =
@@ -3621,7 +3677,7 @@ transformJsonObjectConstructor(ParseState *pstate, JsonObjectConstructor *ctor)
Node *val = transformJsonValueExpr(pstate, "JSON_OBJECT()",
kv->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = lappend(args, key);
args = lappend(args, val);
@@ -3808,7 +3864,7 @@ transformJsonObjectAgg(ParseState *pstate, JsonObjectAgg *agg)
val = transformJsonValueExpr(pstate, "JSON_OBJECTAGG()",
agg->arg->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = list_make2(key, val);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
@@ -3864,9 +3920,8 @@ transformJsonArrayAgg(ParseState *pstate, JsonArrayAgg *agg)
Oid aggfnoid;
Oid aggtype;
- arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()",
- agg->arg,
- JS_FORMAT_DEFAULT, InvalidOid);
+ arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()", agg->arg,
+ JS_FORMAT_DEFAULT, InvalidOid, false);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
list_make1(arg));
@@ -3913,9 +3968,8 @@ transformJsonArrayConstructor(ParseState *pstate, JsonArrayConstructor *ctor)
{
JsonValueExpr *jsval = castNode(JsonValueExpr, lfirst(lc));
Node *val = transformJsonValueExpr(pstate, "JSON_ARRAY()",
- jsval,
- JS_FORMAT_DEFAULT,
- InvalidOid);
+ jsval, JS_FORMAT_DEFAULT,
+ InvalidOid, false);
args = lappend(args, val);
}
@@ -4074,7 +4128,7 @@ transformJsonParseExpr(ParseState *pstate, JsonParseExpr *jsexpr)
* function-like CASTs.
*/
arg = transformJsonValueExpr(pstate, "JSON()", jsexpr->expr,
- JS_FORMAT_JSON, returning->typid);
+ JS_FORMAT_JSON, returning->typid, false);
}
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_PARSE, list_make1(arg), NULL,
@@ -4119,7 +4173,7 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
Node *arg = transformJsonValueExpr(pstate, "JSON_SERIALIZE()",
expr->expr,
JS_FORMAT_JSON,
- InvalidOid);
+ InvalidOid, false);
if (expr->output)
{
@@ -4153,3 +4207,468 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_SERIALIZE, list_make1(arg),
NULL, returning, false, false, expr->location);
}
+
+/*
+ * Transform JSON_VALUE, JSON_QUERY, JSON_EXISTS functions into a JsonExpr node.
+ */
+static Node *
+transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
+{
+ JsonExpr *jsexpr = NULL;
+ const char *func_name = NULL;
+
+ /*
+ * Disallow FORMAT specification in the RETURNING clause of JSON_EXISTS()
+ * and JSON_VALUE().
+ */
+ if (func->output &&
+ (func->op == JSON_VALUE_OP || func->op == JSON_EXISTS_OP))
+ {
+ JsonFormat *format = func->output->returning->format;
+
+ if (format->format_type != JS_FORMAT_DEFAULT ||
+ format->encoding != JS_ENC_DEFAULT)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot specify FORMAT in RETURNING clause of %s",
+ func->op == JSON_VALUE_OP ? "JSON_VALUE()" :
+ "JSON_EXISTS()"),
+ parser_errposition(pstate, format->location)));
+ }
+
+ switch (func->op)
+ {
+ case JSON_EXISTS_OP:
+ func_name = "JSON_EXISTS";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ jsexpr->returning->typid = BOOLOID;
+ jsexpr->returning->typmod = -1;
+ }
+ else if (jsexpr->returning->typid != BOOLOID)
+ {
+ Node *coercion_expr;
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+ int location = exprLocation((Node *) jsexpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the
+ * result of evaluating JSON_EXISTS to the coercion function.
+ */
+ placeholder->typeId = BOOLOID;
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ coercion_expr =
+ coerce_to_target_type(pstate, (Node *) placeholder, BOOLOID,
+ jsexpr->returning->typid,
+ jsexpr->returning->typmod,
+ COERCION_EXPLICIT,
+ COERCE_IMPLICIT_CAST,
+ location);
+
+ if (coercion_expr == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast type %s to %s",
+ format_type_be(BOOLOID),
+ format_type_be(jsexpr->returning->typid)),
+ parser_coercion_errposition(pstate, location, (Node *) jsexpr)));
+
+ if (coercion_expr != (Node *) placeholder)
+ {
+ jsexpr->result_coercion = makeNode(JsonCoercion);
+ jsexpr->result_coercion->expr = coercion_expr;
+ }
+ }
+
+ jsexpr->on_error = transformJsonBehavior(pstate, func->on_error,
+ JSON_BEHAVIOR_FALSE,
+ jsexpr->returning);
+ break;
+
+ case JSON_QUERY_OP:
+ func_name = "JSON_QUERY";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->wrapper = func->wrapper;
+ jsexpr->omit_quotes = (func->quotes == JS_QUOTES_OMIT);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ JsonReturning *ret = jsexpr->returning;
+
+ ret->typid = JsonFuncExprDefaultReturnType(jsexpr);
+ ret->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ jsexpr->on_empty = transformJsonBehavior(pstate, func->on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, func->on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+
+ case JSON_VALUE_OP:
+ func_name = "JSON_VALUE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ /* Make JSON_VALUE return text by default */
+ jsexpr->returning->typid = TEXTOID;
+ jsexpr->returning->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ /*
+ * Initialize JsonCoercion nodes to coerce the scalar value
+ * returned by JsonPathValue() to the "returning" type.
+ */
+ jsexpr->item_coercions =
+ InitJsonItemCoercions(pstate, jsexpr->returning,
+ exprType(jsexpr->formatted_expr));
+
+ jsexpr->on_empty = transformJsonBehavior(pstate, func->on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, func->on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+ }
+
+ Assert(jsexpr != NULL && jsexpr->formatted_expr != NULL);
+ if (exprType(jsexpr->formatted_expr) != JSONBOID)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("%s() is not yet implemented for the json type", func_name),
+ errhint("Try casting the argument to jsonb"),
+ parser_errposition(pstate, func->location)));
+
+ return (Node *) jsexpr;
+}
+
+/*
+ * Common code for JSON_VALUE, JSON_QUERY, JSON_EXISTS transformation
+ * into a JsonExpr node.
+ */
+static JsonExpr *
+transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName)
+{
+ JsonExpr *jsexpr = makeNode(JsonExpr);
+ Node *pathspec;
+
+ jsexpr->location = func->location;
+ jsexpr->op = func->op;
+ jsexpr->formatted_expr = transformJsonValueExpr(pstate, constructName,
+ func->common->expr,
+ JS_FORMAT_DEFAULT,
+ InvalidOid, false);
+
+ jsexpr->format = func->common->expr->format;
+
+ /* Both set in the caller. */
+ jsexpr->result_coercion = NULL;
+ jsexpr->omit_quotes = false;
+
+ pathspec = transformExprRecurse(pstate, func->common->pathspec);
+
+ jsexpr->path_spec =
+ coerce_to_target_type(pstate, pathspec, exprType(pathspec),
+ JSONPATHOID, -1,
+ COERCION_EXPLICIT, COERCE_IMPLICIT_CAST,
+ exprLocation(pathspec));
+ if (!jsexpr->path_spec)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("JSON path expression must be of type %s, not of type %s",
+ "jsonpath", format_type_be(exprType(pathspec))),
+ parser_errposition(pstate, exprLocation(pathspec))));
+
+ /*
+ * Transform and coerce to json[b] passing arguments, whose format is
+ * determined by context item type.
+ */
+ transformJsonPassingArgs(pstate, constructName,
+ exprType(jsexpr->formatted_expr) == JSONBOID ?
+ JS_FORMAT_JSONB : JS_FORMAT_JSON,
+ func->common->passing,
+ &jsexpr->passing_values,
+ &jsexpr->passing_names);
+
+ jsexpr->returning = transformJsonOutput(pstate, func->output, false);
+
+ /* JSON_QUERY support specifying FORMAT explicitly. */
+ if (func->op != JSON_QUERY_OP)
+ {
+ jsexpr->returning->format->format_type = JS_FORMAT_DEFAULT;
+ jsexpr->returning->format->encoding = JS_ENC_DEFAULT;
+ }
+
+ return jsexpr;
+}
+
+/*
+ * Transform a JSON PASSING clause.
+ */
+static void
+transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names)
+{
+ ListCell *lc;
+
+ *passing_values = NIL;
+ *passing_names = NIL;
+
+ foreach(lc, args)
+ {
+ JsonArgument *arg = castNode(JsonArgument, lfirst(lc));
+ Node *expr = transformJsonValueExpr(pstate, constructName,
+ arg->val, format,
+ InvalidOid, true);
+
+ *passing_values = lappend(*passing_values, expr);
+ *passing_names = lappend(*passing_names, makeString(arg->name));
+ }
+}
+
+/*
+ * Transform the JSON output clause of JSON_VALUE and JSON_QUERY.
+ */
+static JsonCoercion *
+coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr)
+{
+ JsonCoercion *coercion = NULL;
+ JsonReturning *returning = jsexpr->returning;
+ Node *context_item = jsexpr->formatted_expr;
+ int default_typmod;
+ Oid default_typid;
+
+ Assert(returning);
+
+ /*
+ * Forced coercion via I/O for non-JSON types, except for JSON_QUERY()
+ * which must implement the specified JsonQuotes behavior.
+ */
+ if (returning->typid != JSONOID && returning->typid != JSONBOID &&
+ (jsexpr->op != JSON_QUERY_OP || jsexpr->omit_quotes))
+ {
+ coercion = makeNode(JsonCoercion);
+ coercion->expr = NULL;
+ coercion->via_io = true;
+
+ return coercion;
+ }
+ else if (jsexpr->op == JSON_QUERY_OP && jsexpr->wrapper != JSW_NONE)
+ {
+ coercion = makeNode(JsonCoercion);
+ coercion->expr = NULL;
+ coercion->via_populate = true;
+
+ return coercion;
+ }
+
+ default_typid = JsonFuncExprDefaultReturnType(jsexpr);
+ default_typmod = -1;
+ if (returning->typid != default_typid ||
+ returning->typmod != default_typmod)
+ {
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * evaluating the JSON_VALUE/QUERY jsonpath expression to the coercion
+ * function.
+ */
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ placeholder->typeId = exprType(context_item);
+ placeholder->typeMod = exprTypmod(context_item);
+ placeholder->collation = exprCollation(context_item);
+
+ Assert(placeholder->typeId == default_typid);
+ Assert(placeholder->typeMod == default_typmod);
+
+ coercion = coerceJsonExpr(pstate, (Node *) placeholder, returning);
+ }
+
+ return coercion;
+}
+
+/* Returns the default type for a given JsonExpr for a given JsonFormat. */
+static Oid
+JsonFuncExprDefaultReturnType(JsonExpr *jsexpr)
+{
+ JsonFormat *format = jsexpr->format;
+ Node *context_item = jsexpr->formatted_expr;
+
+ Assert(format);
+ if (format->format_type == JS_FORMAT_JSONB)
+ return JSONBOID;
+ else if (format->format_type == JS_FORMAT_DEFAULT &&
+ exprType(context_item) == JSONBOID)
+ return JSONBOID;
+
+ return JSONOID;
+}
+
+/*
+ * Set up a JsonCoercion node to:
+ *
+ * - coerce expression to the output returning type, or
+ * - coerce using json_populate_type() if returning type requires it, or
+ * - coerce via I/O.
+ */
+static JsonCoercion *
+coerceJsonExpr(ParseState *pstate, Node *expr, const JsonReturning *returning)
+{
+ char typtype;
+ JsonCoercion *coercion = makeNode(JsonCoercion);
+
+ coercion->expr = coerceJsonFuncExpr(pstate, expr, returning, false);
+ if (coercion->expr)
+ {
+ if (coercion->expr == expr)
+ coercion->expr = NULL;
+
+ return coercion;
+ }
+
+ typtype = get_typtype(returning->typid);
+ if (returning->typid == RECORDOID ||
+ typtype == TYPTYPE_COMPOSITE ||
+ typtype == TYPTYPE_DOMAIN ||
+ type_is_array(returning->typid))
+ coercion->via_populate = true;
+ else
+ coercion->via_io = true;
+
+ return coercion;
+}
+
+/*
+ * Initialize JsonCoercion nodes for coercing a given JSON item value produced
+ * by JSON_VALUE to the target "returning" type; also see
+ * ExecPrepareJsonItemCoercion().
+ */
+static List *
+InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId)
+{
+ List *item_coercions = NIL;
+ int i;
+ Oid item_typeoids[] =
+ {
+ UNKNOWNOID,
+ TEXTOID,
+ NUMERICOID,
+ BOOLOID,
+ DATEOID,
+ TIMEOID,
+ TIMETZOID,
+ TIMESTAMPOID,
+ TIMESTAMPTZOID,
+ contextItemTypeId,
+ InvalidOid
+ };
+
+ for (i = 0; item_typeoids[i] != InvalidOid; i++)
+ {
+ Node *expr;
+ JsonCoercion *coercion;
+
+ if (item_typeoids[i] == UNKNOWNOID)
+ {
+ expr = (Node *) makeNullConst(UNKNOWNOID, -1, InvalidOid);
+ }
+ else
+ {
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * JSON_VALUE jsonpath expression to the coercion function.
+ */
+ placeholder->typeId = item_typeoids[i];
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ expr = (Node *) placeholder;
+ }
+
+ coercion = coerceJsonExpr(pstate, expr, returning);
+ item_coercions = lappend(item_coercions, coercion);
+ }
+
+ return item_coercions;
+}
+
+/*
+ * Transform a JSON BEHAVIOR clause.
+ */
+static JsonBehavior *
+transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning)
+{
+ JsonBehaviorType behavior_type = default_behavior;
+ Node *default_expr = NULL;
+
+ if (behavior)
+ {
+ behavior_type = behavior->btype;
+ if (behavior_type == JSON_BEHAVIOR_DEFAULT)
+ default_expr = transformExprRecurse(pstate, behavior->default_expr);
+ }
+
+ behavior = makeJsonBehavior(behavior_type, default_expr, -1);
+
+ /*
+ * Also coerce the DEFAULT expression, if any, to match the returning
+ * type.
+ */
+ return coerceJsonBehaviorDefaultExpr(pstate, behavior, returning);
+}
+
+/*
+ * Coerce expression in `DEFAULT expression ON ERROR / EMPTY` to the target
+ * output type.
+ */
+static JsonBehavior *
+coerceJsonBehaviorDefaultExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning)
+{
+ Oid exprtype;
+
+ if (behavior->default_expr == NULL)
+ return behavior;
+
+ exprtype = exprType(behavior->default_expr);
+
+ behavior->default_expr =
+ coerce_to_target_type(pstate,
+ behavior->default_expr,
+ exprtype,
+ returning->typid,
+ returning->typmod,
+ COERCION_EXPLICIT,
+ COERCE_IMPLICIT_CAST,
+ exprLocation((Node *) behavior));
+
+ if (behavior->default_expr == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast DEFAULT expression of type %s to %s",
+ format_type_be(exprtype),
+ format_type_be(returning->typid)),
+ parser_errposition(pstate, exprLocation((Node *) behavior))));
+
+ return behavior;
+}
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 3bc62ac3ba..8f81624f97 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -1989,6 +1989,21 @@ FigureColnameInternal(Node *node, char **name)
/* make JSON_ARRAYAGG act like a regular function */
*name = "json_arrayagg";
return 2;
+ case T_JsonFuncExpr:
+ /* make SQL/JSON functions act like a regular function */
+ switch (((JsonFuncExpr *) node)->op)
+ {
+ case JSON_EXISTS_OP:
+ *name = "json_exists";
+ return 2;
+ case JSON_QUERY_OP:
+ *name = "json_query";
+ return 2;
+ case JSON_VALUE_OP:
+ *name = "json_value";
+ return 2;
+ }
+ break;
default:
break;
}
diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c
index e27ea8ef97..188b5594e0 100644
--- a/src/backend/utils/adt/formatting.c
+++ b/src/backend/utils/adt/formatting.c
@@ -4426,6 +4426,50 @@ parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
}
}
+/*
+ * Parses the datetime format string in 'fmt_str' and returns true if it
+ * contains a timezone specifier, false if not.
+ */
+bool
+datetime_format_has_tz(const char *fmt_str)
+{
+ bool incache;
+ int fmt_len = strlen(fmt_str);
+ int result;
+ FormatNode *format;
+
+ if (fmt_len > DCH_CACHE_SIZE)
+ {
+ /*
+ * Allocate new memory if format picture is bigger than static cache
+ * and do not use cache (call parser always)
+ */
+ incache = false;
+
+ format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode));
+
+ parse_format(format, fmt_str, DCH_keywords,
+ DCH_suff, DCH_index, DCH_FLAG, NULL);
+ }
+ else
+ {
+ /*
+ * Use cache buffers
+ */
+ DCHCacheEntry *ent = DCH_cache_fetch(fmt_str, false);
+
+ incache = true;
+ format = ent->format;
+ }
+
+ result = DCH_datetime_type(format);
+
+ if (!incache)
+ pfree(format);
+
+ return result & DCH_ZONED;
+}
+
/*
* do_to_timestamp: shared code for to_timestamp and to_date
*
diff --git a/src/backend/utils/adt/jsonb.c b/src/backend/utils/adt/jsonb.c
index 9781852b0c..ea5b386f8c 100644
--- a/src/backend/utils/adt/jsonb.c
+++ b/src/backend/utils/adt/jsonb.c
@@ -2162,3 +2162,65 @@ jsonb_float8(PG_FUNCTION_ARGS)
PG_RETURN_DATUM(retValue);
}
+
+/*
+ * Construct an empty array jsonb.
+ */
+Jsonb *
+JsonbMakeEmptyArray(void)
+{
+ JsonbValue jbv;
+
+ jbv.type = jbvArray;
+ jbv.val.array.elems = NULL;
+ jbv.val.array.nElems = 0;
+ jbv.val.array.rawScalar = false;
+
+ return JsonbValueToJsonb(&jbv);
+}
+
+/*
+ * Construct an empty object jsonb.
+ */
+Jsonb *
+JsonbMakeEmptyObject(void)
+{
+ JsonbValue jbv;
+
+ jbv.type = jbvObject;
+ jbv.val.object.pairs = NULL;
+ jbv.val.object.nPairs = 0;
+
+ return JsonbValueToJsonb(&jbv);
+}
+
+/*
+ * Convert jsonb to a C-string stripping quotes from scalar strings.
+ */
+char *
+JsonbUnquote(Jsonb *jb)
+{
+ if (JB_ROOT_IS_SCALAR(jb))
+ {
+ JsonbValue v;
+
+ (void) JsonbExtractScalar(&jb->root, &v);
+
+ if (v.type == jbvString)
+ return pnstrdup(v.val.string.val, v.val.string.len);
+ else if (v.type == jbvBool)
+ return pstrdup(v.val.boolean ? "true" : "false");
+ else if (v.type == jbvNumeric)
+ return DatumGetCString(DirectFunctionCall1(numeric_out,
+ PointerGetDatum(v.val.numeric)));
+ else if (v.type == jbvNull)
+ return pstrdup("null");
+ else
+ {
+ elog(ERROR, "unrecognized jsonb value type %d", v.type);
+ return NULL;
+ }
+ }
+ else
+ return JsonbToCString(NULL, &jb->root, VARSIZE(jb));
+}
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index a8f2b186c9..596d935f10 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2804,7 +2804,8 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
- if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc) ||
+ JsonContainerIsScalar(jbc))
{
populate_array_report_expected_array(ctx, ndim - 1);
/* Getting here means the error was reported softly. */
@@ -2812,8 +2813,6 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
return false;
}
- Assert(!JsonContainerIsScalar(jbc));
-
it = JsonbIteratorInit(jbc);
tok = JsonbIteratorNext(&it, &val, true);
@@ -3348,6 +3347,53 @@ populate_record_field(ColumnIOData *col,
}
}
+/* recursively populate specified type from a json/jsonb value */
+Datum
+json_populate_type(Datum json_val, Oid json_type, Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext)
+{
+ JsValue jsv = {0};
+ JsonbValue jbv;
+
+ jsv.is_json = json_type == JSONOID;
+
+ if (*isnull)
+ {
+ if (jsv.is_json)
+ jsv.val.json.str = NULL;
+ else
+ jsv.val.jsonb = NULL;
+ }
+ else if (jsv.is_json)
+ {
+ text *json = DatumGetTextPP(json_val);
+
+ jsv.val.json.str = VARDATA_ANY(json);
+ jsv.val.json.len = VARSIZE_ANY_EXHDR(json);
+ jsv.val.json.type = JSON_TOKEN_INVALID; /* not used in
+ * populate_composite() */
+ }
+ else
+ {
+ Jsonb *jsonb = DatumGetJsonbP(json_val);
+
+ jsv.val.jsonb = &jbv;
+
+ /* fill binary jsonb value pointing to jb */
+ jbv.type = jbvBinary;
+ jbv.val.binary.data = &jsonb->root;
+ jbv.val.binary.len = VARSIZE(jsonb) - VARHDRSZ;
+ }
+
+ if (!*cache)
+ *cache = MemoryContextAllocZero(mcxt, sizeof(ColumnIOData));
+
+ return populate_record_field(*cache, typid, typmod, NULL, mcxt,
+ PointerGetDatum(NULL), &jsv, isnull,
+ escontext);
+}
+
static RecordIOData *
allocate_record_info(MemoryContext mcxt, int ncolumns)
{
diff --git a/src/backend/utils/adt/jsonpath.c b/src/backend/utils/adt/jsonpath.c
index c5ba3b7f1d..8d89994c10 100644
--- a/src/backend/utils/adt/jsonpath.c
+++ b/src/backend/utils/adt/jsonpath.c
@@ -68,7 +68,9 @@
#include "libpq/pqformat.h"
#include "nodes/miscnodes.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/builtins.h"
+#include "utils/formatting.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
@@ -1110,3 +1112,256 @@ jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from, JsonPathItem *to,
return true;
}
+
+/* SQL/JSON datatype status: */
+typedef enum JsonPathDatatypeStatus
+{
+ jpdsNonDateTime, /* null, bool, numeric, string, array, object */
+ jpdsUnknownDateTime, /* unknown datetime type */
+ jpdsDateTimeZoned, /* timetz, timestamptz */
+ jpdsDateTimeNonZoned /* time, timestamp, date */
+} JsonPathDatatypeStatus;
+
+/* Context for jspIsMutableWalker() */
+typedef struct JsonPathMutableContext
+{
+ List *varnames; /* list of variable names */
+ List *varexprs; /* list of variable expressions */
+ JsonPathDatatypeStatus current; /* status of @ item */
+ bool lax; /* jsonpath is lax or strict */
+ bool mutable; /* resulting mutability status */
+} JsonPathMutableContext;
+
+/*
+ * Recursive walker for jspIsMutable()
+ */
+static JsonPathDatatypeStatus
+jspIsMutableWalker(JsonPathItem *jpi, JsonPathMutableContext *cxt)
+{
+ JsonPathItem next;
+ JsonPathDatatypeStatus status = jpdsNonDateTime;
+
+ while (!cxt->mutable)
+ {
+ JsonPathItem arg;
+ JsonPathDatatypeStatus leftStatus;
+ JsonPathDatatypeStatus rightStatus;
+
+ switch (jpi->type)
+ {
+ case jpiRoot:
+ Assert(status == jpdsNonDateTime);
+ break;
+
+ case jpiCurrent:
+ Assert(status == jpdsNonDateTime);
+ status = cxt->current;
+ break;
+
+ case jpiFilter:
+ {
+ JsonPathDatatypeStatus prevStatus = cxt->current;
+
+ cxt->current = status;
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+
+ cxt->current = prevStatus;
+ break;
+ }
+
+ case jpiVariable:
+ {
+ int32 len;
+ const char *name = jspGetString(jpi, &len);
+ ListCell *lc1;
+ ListCell *lc2;
+
+ Assert(status == jpdsNonDateTime);
+
+ forboth(lc1, cxt->varnames, lc2, cxt->varexprs)
+ {
+ String *varname = lfirst_node(String, lc1);
+ Node *varexpr = lfirst(lc2);
+
+ if (strncmp(varname->sval, name, len))
+ continue;
+
+ switch (exprType(varexpr))
+ {
+ case DATEOID:
+ case TIMEOID:
+ case TIMESTAMPOID:
+ status = jpdsDateTimeNonZoned;
+ break;
+
+ case TIMETZOID:
+ case TIMESTAMPTZOID:
+ status = jpdsDateTimeZoned;
+ break;
+
+ default:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ break;
+ }
+ break;
+ }
+
+ case jpiEqual:
+ case jpiNotEqual:
+ case jpiLess:
+ case jpiGreater:
+ case jpiLessOrEqual:
+ case jpiGreaterOrEqual:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ leftStatus = jspIsMutableWalker(&arg, cxt);
+
+ jspGetRightArg(jpi, &arg);
+ rightStatus = jspIsMutableWalker(&arg, cxt);
+
+ /*
+ * Comparison of datetime type with different timezone status
+ * is mutable.
+ */
+ if (leftStatus != jpdsNonDateTime &&
+ rightStatus != jpdsNonDateTime &&
+ (leftStatus == jpdsUnknownDateTime ||
+ rightStatus == jpdsUnknownDateTime ||
+ leftStatus != rightStatus))
+ cxt->mutable = true;
+ break;
+
+ case jpiNot:
+ case jpiIsUnknown:
+ case jpiExists:
+ case jpiPlus:
+ case jpiMinus:
+ Assert(status == jpdsNonDateTime);
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiAnd:
+ case jpiOr:
+ case jpiAdd:
+ case jpiSub:
+ case jpiMul:
+ case jpiDiv:
+ case jpiMod:
+ case jpiStartsWith:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ jspGetRightArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiIndexArray:
+ for (int i = 0; i < jpi->content.array.nelems; i++)
+ {
+ JsonPathItem from;
+ JsonPathItem to;
+
+ if (jspGetArraySubscript(jpi, &from, &to, i))
+ jspIsMutableWalker(&to, cxt);
+
+ jspIsMutableWalker(&from, cxt);
+ }
+ /* FALLTHROUGH */
+
+ case jpiAnyArray:
+ if (!cxt->lax)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiAny:
+ if (jpi->content.anybounds.first > 0)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiDatetime:
+ if (jpi->content.arg)
+ {
+ char *template;
+
+ jspGetArg(jpi, &arg);
+ if (arg.type != jpiString)
+ {
+ status = jpdsNonDateTime;
+ break; /* there will be runtime error */
+ }
+
+ template = jspGetString(&arg, NULL);
+ if (datetime_format_has_tz(template))
+ status = jpdsDateTimeZoned;
+ else
+ status = jpdsDateTimeNonZoned;
+ }
+ else
+ {
+ status = jpdsUnknownDateTime;
+ }
+ break;
+
+ case jpiLikeRegex:
+ Assert(status == jpdsNonDateTime);
+ jspInitByBuffer(&arg, jpi->base, jpi->content.like_regex.expr);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ /* literals */
+ case jpiNull:
+ case jpiString:
+ case jpiNumeric:
+ case jpiBool:
+ /* accessors */
+ case jpiKey:
+ case jpiAnyKey:
+ /* special items */
+ case jpiSubscript:
+ case jpiLast:
+ /* item methods */
+ case jpiType:
+ case jpiSize:
+ case jpiAbs:
+ case jpiFloor:
+ case jpiCeiling:
+ case jpiDouble:
+ case jpiKeyValue:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ if (!jspGetNext(jpi, &next))
+ break;
+
+ jpi = &next;
+ }
+
+ return status;
+}
+
+/*
+ * Check whether jsonpath expression is immutable or not.
+ */
+bool
+jspIsMutable(JsonPath *path, List *varnames, List *varexprs)
+{
+ JsonPathMutableContext cxt;
+ JsonPathItem jpi;
+
+ cxt.varnames = varnames;
+ cxt.varexprs = varexprs;
+ cxt.current = jpdsNonDateTime;
+ cxt.lax = (path->header & JSONPATH_LAX) != 0;
+ cxt.mutable = false;
+
+ jspInit(&jpi, path);
+ jspIsMutableWalker(&jpi, &cxt);
+
+ return cxt.mutable;
+}
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 2d0599b4aa..6c5602c64d 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -87,12 +87,16 @@ typedef struct JsonBaseObjectInfo
int id;
} JsonBaseObjectInfo;
+typedef int (*JsonPathVarCallback) (void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
+
/*
* Context of jsonpath execution.
*/
typedef struct JsonPathExecContext
{
- Jsonb *vars; /* variables to substitute into jsonpath */
+ void *vars; /* variables to substitute into jsonpath */
+ JsonPathVarCallback getVar;
JsonbValue *root; /* for $ evaluation */
JsonbValue *current; /* for @ evaluation */
JsonBaseObjectInfo baseObject; /* "base object" for .keyvalue()
@@ -174,7 +178,8 @@ typedef JsonPathBool (*JsonPathPredicateCallback) (JsonPathItem *jsp,
void *param);
typedef Numeric (*BinaryArithmFunc) (Numeric num1, Numeric num2, bool *error);
-static JsonPathExecResult executeJsonPath(JsonPath *path, Jsonb *vars,
+static JsonPathExecResult executeJsonPath(JsonPath *path, void *vars,
+ JsonPathVarCallback getVar,
Jsonb *json, bool throwErrors,
JsonValueList *result, bool useTz);
static JsonPathExecResult executeItem(JsonPathExecContext *cxt,
@@ -225,8 +230,13 @@ static JsonPathExecResult appendBoolResult(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonValueList *found, JsonPathBool res);
static void getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
JsonbValue *value);
+static int GetJsonPathVar(void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
static void getJsonPathVariable(JsonPathExecContext *cxt,
- JsonPathItem *variable, Jsonb *vars, JsonbValue *value);
+ JsonPathItem *variable, JsonbValue *value);
+static int getJsonPathVariableFromJsonb(void *varsJsonb, char *varName,
+ int varNameLen, JsonbValue *val,
+ JsonbValue *baseObject);
static int JsonbArraySize(JsonbValue *jb);
static JsonPathBool executeComparison(JsonPathItem *cmp, JsonbValue *lv,
JsonbValue *rv, void *p);
@@ -284,7 +294,8 @@ jsonb_path_exists_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- res = executeJsonPath(jp, vars, jb, !silent, NULL, tz);
+ res = executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, NULL, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -339,7 +350,8 @@ jsonb_path_match_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -417,7 +429,8 @@ jsonb_path_query_internal(FunctionCallInfo fcinfo, bool tz)
vars = PG_GETARG_JSONB_P_COPY(2);
silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
funcctx->user_fctx = JsonValueListGetList(&found);
@@ -464,7 +477,8 @@ jsonb_path_query_array_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_RETURN_JSONB_P(JsonbValueToJsonb(wrapItemsInArray(&found)));
}
@@ -495,7 +509,8 @@ jsonb_path_query_first_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
if (JsonValueListLength(&found) >= 1)
PG_RETURN_JSONB_P(JsonbValueToJsonb(JsonValueListHead(&found)));
@@ -537,8 +552,9 @@ jsonb_path_query_first_tz(PG_FUNCTION_ARGS)
* In other case it tries to find all the satisfied result items.
*/
static JsonPathExecResult
-executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
- JsonValueList *result, bool useTz)
+executeJsonPath(JsonPath *path, void *vars, JsonPathVarCallback getVar,
+ Jsonb *json, bool throwErrors, JsonValueList *result,
+ bool useTz)
{
JsonPathExecContext cxt;
JsonPathExecResult res;
@@ -550,22 +566,16 @@ executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
if (!JsonbExtractScalar(&json->root, &jbv))
JsonbInitBinary(&jbv, json);
- if (vars && !JsonContainerIsObject(&vars->root))
- {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("\"vars\" argument is not an object"),
- errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
- }
-
cxt.vars = vars;
+ cxt.getVar = getVar;
cxt.laxMode = (path->header & JSONPATH_LAX) != 0;
cxt.ignoreStructuralErrors = cxt.laxMode;
cxt.root = &jbv;
cxt.current = &jbv;
cxt.baseObject.jbc = NULL;
cxt.baseObject.id = 0;
- cxt.lastGeneratedObjectId = vars ? 2 : 1;
+ /* 1 + number of base objects in vars */
+ cxt.lastGeneratedObjectId = 1 + getVar(vars, NULL, 0, NULL, NULL);
cxt.innermostArraySize = -1;
cxt.throwErrors = throwErrors;
cxt.useTz = useTz;
@@ -2108,54 +2118,118 @@ getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
&value->val.string.len);
break;
case jpiVariable:
- getJsonPathVariable(cxt, item, cxt->vars, value);
+ getJsonPathVariable(cxt, item, value);
return;
default:
elog(ERROR, "unexpected jsonpath item type");
}
}
+/*
+ * Returns the computed value of a JSON path variable with given name.
+ */
+static int
+GetJsonPathVar(void *cxt, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject)
+{
+ JsonPathVariable *var = NULL;
+ List *vars = cxt;
+ ListCell *lc;
+ int id = 1;
+
+ if (!varName)
+ return list_length(vars);
+
+ foreach(lc, vars)
+ {
+ JsonPathVariable *curvar = lfirst(lc);
+
+ if (!strncmp(curvar->name, varName, varNameLen))
+ {
+ var = curvar;
+ break;
+ }
+
+ id++;
+ }
+
+ if (!var)
+ return -1;
+
+ if (var->isnull)
+ {
+ val->type = jbvNull;
+ return 0;
+ }
+
+ JsonItemFromDatum(var->value, var->typid, var->typmod, val);
+
+ *baseObject = *val;
+ return id;
+}
+
/*
* Get the value of variable passed to jsonpath executor
*/
static void
getJsonPathVariable(JsonPathExecContext *cxt, JsonPathItem *variable,
- Jsonb *vars, JsonbValue *value)
+ JsonbValue *value)
{
char *varName;
int varNameLength;
+ JsonbValue baseObject;
+ int baseObjectId;
+
+ Assert(variable->type == jpiVariable);
+ varName = jspGetString(variable, &varNameLength);
+
+ if (!cxt->vars ||
+ (baseObjectId = cxt->getVar(cxt->vars, varName, varNameLength, value,
+ &baseObject)) < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("could not find jsonpath variable \"%s\"",
+ pnstrdup(varName, varNameLength))));
+
+ if (baseObjectId > 0)
+ setBaseObject(cxt, &baseObject, baseObjectId);
+}
+
+static int
+getJsonPathVariableFromJsonb(void *varsJsonb, char *varName, int varNameLength,
+ JsonbValue *value, JsonbValue *baseObject)
+{
+ Jsonb *vars = varsJsonb;
JsonbValue tmp;
JsonbValue *v;
- if (!vars)
+ if (!varName)
{
- value->type = jbvNull;
- return;
+ if (vars && !JsonContainerIsObject(&vars->root))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"vars\" argument is not an object"),
+ errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
+ }
+
+ return vars ? 1 : 0; /* count of base objects */
}
- Assert(variable->type == jpiVariable);
- varName = jspGetString(variable, &varNameLength);
tmp.type = jbvString;
tmp.val.string.val = varName;
tmp.val.string.len = varNameLength;
v = findJsonbValueFromContainer(&vars->root, JB_FOBJECT, &tmp);
- if (v)
- {
- *value = *v;
- pfree(v);
- }
- else
- {
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_OBJECT),
- errmsg("could not find jsonpath variable \"%s\"",
- pnstrdup(varName, varNameLength))));
- }
+ if (!v)
+ return -1;
- JsonbInitBinary(&tmp, vars);
- setBaseObject(cxt, &tmp, 1);
+ *value = *v;
+ pfree(v);
+
+ JsonbInitBinary(baseObject, vars);
+ return 1;
}
/**************** Support functions for JsonPath execution *****************/
@@ -2812,3 +2886,240 @@ compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
return DatumGetInt32(DirectFunctionCall2(cmpfunc, val1, val2));
}
+
+/* Executor-callable JSON_EXISTS implementation */
+bool
+JsonPathExists(Datum jb, JsonPath *jp, bool *error, List *vars)
+{
+ JsonPathExecResult res = executeJsonPath(jp, vars, GetJsonPathVar,
+ DatumGetJsonbP(jb), !error, NULL,
+ true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ *error = true;
+
+ return res == jperOk;
+}
+
+/* Executor-callable JSON_QUERY implementation */
+Datum
+JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper, bool *empty,
+ bool *error, List *vars)
+{
+ JsonbValue *first;
+ bool wrap;
+ JsonValueList found = {0};
+ JsonPathExecResult res PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ res = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ {
+ *error = true;
+ *empty = false;
+ return (Datum) 0;
+ }
+
+ count = JsonValueListLength(&found);
+
+ first = count > 0 ? JsonValueListHead(&found) : NULL;
+
+ if (!first)
+ wrap = false;
+ else if (wrapper == JSW_NONE)
+ wrap = false;
+ else if (wrapper == JSW_UNCONDITIONAL)
+ wrap = true;
+ else if (wrapper == JSW_CONDITIONAL)
+ wrap = count > 1 ||
+ IsAJsonbScalar(first) ||
+ (first->type == jbvBinary &&
+ JsonContainerIsScalar(first->val.binary.data));
+ else
+ {
+ elog(ERROR, "unrecognized json wrapper %d", wrapper);
+ wrap = false;
+ }
+
+ if (wrap)
+ return JsonbPGetDatum(JsonbValueToJsonb(wrapItemsInArray(&found)));
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return (Datum) 0;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_QUERY should return singleton item without wrapper"),
+ errhint("Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.")));
+ }
+
+ if (first)
+ return JsonbPGetDatum(JsonbValueToJsonb(first));
+
+ *empty = true;
+ return PointerGetDatum(NULL);
+}
+
+/* Executor-callable JSON_VALUE implementation */
+JsonbValue *
+JsonPathValue(Datum jb, JsonPath *jp, bool *empty, bool *error, List *vars)
+{
+ JsonbValue *res;
+ JsonValueList found = {0};
+ JsonPathExecResult jper PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ jper = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(jper));
+
+ if (error && jperIsError(jper))
+ {
+ *error = true;
+ *empty = false;
+ return NULL;
+ }
+
+ count = JsonValueListLength(&found);
+
+ *empty = (count == 0);
+
+ if (*empty)
+ return NULL;
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ res = JsonValueListHead(&found);
+
+ if (res->type == jbvBinary &&
+ JsonContainerIsScalar(res->val.binary.data))
+ JsonbExtractScalar(res->val.binary.data, res);
+
+ if (!IsAJsonbScalar(res))
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_SCALAR_REQUIRED),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ if (res->type == jbvNull)
+ return NULL;
+
+ return res;
+}
+
+static void
+JsonbValueInitNumericDatum(JsonbValue *jbv, Datum num)
+{
+ jbv->type = jbvNumeric;
+ jbv->val.numeric = DatumGetNumeric(num);
+}
+
+void
+JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
+{
+ switch (typid)
+ {
+ case BOOLOID:
+ res->type = jbvBool;
+ res->val.boolean = DatumGetBool(val);
+ break;
+ case NUMERICOID:
+ JsonbValueInitNumericDatum(res, val);
+ break;
+ case INT2OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int2_numeric, val));
+ break;
+ case INT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int4_numeric, val));
+ break;
+ case INT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int8_numeric, val));
+ break;
+ case FLOAT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float4_numeric, val));
+ break;
+ case FLOAT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float8_numeric, val));
+ break;
+ case TEXTOID:
+ case VARCHAROID:
+ res->type = jbvString;
+ res->val.string.val = VARDATA_ANY(val);
+ res->val.string.len = VARSIZE_ANY_EXHDR(val);
+ break;
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ res->type = jbvDatetime;
+ res->val.datetime.value = val;
+ res->val.datetime.typid = typid;
+ res->val.datetime.typmod = typmod;
+ res->val.datetime.tz = 0;
+ break;
+ case JSONBOID:
+ {
+ JsonbValue *jbv = res;
+ Jsonb *jb = DatumGetJsonbP(val);
+
+ if (JsonContainerIsScalar(&jb->root))
+ {
+ bool result PG_USED_FOR_ASSERTS_ONLY;
+
+ result = JsonbExtractScalar(&jb->root, jbv);
+ Assert(result);
+ }
+ else
+ JsonbInitBinary(jbv, jb);
+ break;
+ }
+ case JSONOID:
+ {
+ text *txt = DatumGetTextP(val);
+ char *str = text_to_cstring(txt);
+ Jsonb *jb;
+
+ jb = DatumGetJsonbP(DirectFunctionCall1(jsonb_in,
+ CStringGetDatum(str)));
+ pfree(str);
+
+ JsonItemFromDatum(JsonbPGetDatum(jb), JSONBOID, -1, res);
+ break;
+ }
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
+ }
+}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 8d5eac4791..d20b96780c 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -476,6 +476,8 @@ static void get_const_expr(Const *constval, deparse_context *context,
int showtype);
static void get_const_collation(Const *constval, deparse_context *context);
static void get_json_format(JsonFormat *format, StringInfo buf);
+static void get_json_returning(JsonReturning *returning, StringInfo buf,
+ bool json_format_by_default);
static void get_json_constructor(JsonConstructorExpr *ctor,
deparse_context *context, bool showimplicit);
static void get_json_constructor_options(JsonConstructorExpr *ctor,
@@ -518,6 +520,8 @@ static char *generate_qualified_type_name(Oid typid);
static text *string_to_text(char *str);
static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
+static void get_json_path_spec(Node *path_spec, deparse_context *context,
+ bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8300,6 +8304,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_WindowFunc:
case T_FuncExpr:
case T_JsonConstructorExpr:
+ case T_JsonExpr:
/* function-like: name(..) or name[..] */
return true;
@@ -8471,6 +8476,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_GroupingFunc: /* own parentheses */
case T_WindowFunc: /* own parentheses */
case T_CaseExpr: /* other separators */
+ case T_JsonExpr: /* own parentheses */
return true;
default:
return false;
@@ -8586,6 +8592,65 @@ get_rule_expr_paren(Node *node, deparse_context *context,
appendStringInfoChar(context->buf, ')');
}
+static void
+get_json_behavior(JsonBehavior *behavior, deparse_context *context,
+ const char *on)
+{
+ /*
+ * The order of array elements must correspond to the order of
+ * JsonBehaviorType members.
+ */
+ const char *behavior_names[] =
+ {
+ " NULL",
+ " ERROR",
+ " EMPTY",
+ " TRUE",
+ " FALSE",
+ " UNKNOWN",
+ " EMPTY ARRAY",
+ " EMPTY OBJECT",
+ " DEFAULT "
+ };
+
+ if ((int) behavior->btype < 0 || behavior->btype >= lengthof(behavior_names))
+ elog(ERROR, "invalid json behavior type: %d", behavior->btype);
+
+ appendStringInfoString(context->buf, behavior_names[behavior->btype]);
+
+ if (behavior->btype == JSON_BEHAVIOR_DEFAULT)
+ get_rule_expr(behavior->default_expr, context, false);
+
+ appendStringInfo(context->buf, " ON %s", on);
+}
+
+/*
+ * get_json_expr_options
+ *
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ */
+static void
+get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
+ JsonBehaviorType default_behavior)
+{
+ if (jsexpr->op == JSON_QUERY_OP)
+ {
+ if (jsexpr->wrapper == JSW_CONDITIONAL)
+ appendStringInfo(context->buf, " WITH CONDITIONAL WRAPPER");
+ else if (jsexpr->wrapper == JSW_UNCONDITIONAL)
+ appendStringInfo(context->buf, " WITH UNCONDITIONAL WRAPPER");
+
+ if (jsexpr->omit_quotes)
+ appendStringInfo(context->buf, " OMIT QUOTES");
+ }
+
+ if (jsexpr->op != JSON_EXISTS_OP &&
+ jsexpr->on_empty->btype != default_behavior)
+ get_json_behavior(jsexpr->on_empty, context, "EMPTY");
+
+ if (jsexpr->on_error->btype != default_behavior)
+ get_json_behavior(jsexpr->on_error, context, "ERROR");
+}
/* ----------
* get_rule_expr - Parse back an expression
@@ -9745,6 +9810,7 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+
case T_JsonValueExpr:
{
JsonValueExpr *jve = (JsonValueExpr *) node;
@@ -9794,6 +9860,64 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ appendStringInfoString(buf, "JSON_EXISTS(");
+ break;
+ case JSON_QUERY_OP:
+ appendStringInfoString(buf, "JSON_QUERY(");
+ break;
+ case JSON_VALUE_OP:
+ appendStringInfoString(buf, "JSON_VALUE(");
+ break;
+ }
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_json_path_spec(jexpr->path_spec, context, showimplicit);
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoString(buf, " PASSING ");
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ get_rule_expr((Node *) lfirst(lc2), context, showimplicit);
+ appendStringInfo(buf, " AS %s",
+ ((String *) lfirst_node(String, lc1))->sval);
+ }
+ }
+
+ if (jexpr->op != JSON_EXISTS_OP ||
+ jexpr->returning->typid != BOOLOID)
+ get_json_returning(jexpr->returning, context->buf,
+ jexpr->op == JSON_QUERY_OP);
+
+ get_json_expr_options(jexpr, context,
+ jexpr->op != JSON_EXISTS_OP ?
+ JSON_BEHAVIOR_NULL :
+ JSON_BEHAVIOR_FALSE);
+
+ appendStringInfoString(buf, ")");
+ }
+ break;
+
case T_List:
{
char *sep;
@@ -9917,6 +10041,7 @@ looks_like_function(Node *node)
case T_MinMaxExpr:
case T_SQLValueFunction:
case T_XmlExpr:
+ case T_JsonExpr:
/* these are all accepted by func_expr_common_subexpr */
return true;
default:
@@ -10776,6 +10901,18 @@ get_const_collation(Const *constval, deparse_context *context)
}
}
+/*
+ * get_json_path_spec - Parse back a JSON path specification
+ */
+static void
+get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit)
+{
+ if (IsA(path_spec, Const))
+ get_const_expr((Const *) path_spec, context, -1);
+ else
+ get_rule_expr(path_spec, context, showimplicit);
+}
+
/*
* get_json_format - Parse back a JsonFormat node
*/
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index 59f3b043c6..1e95a3ab22 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -23,6 +23,8 @@ struct ExprEvalStep;
struct SubscriptingRefState;
struct ScalarArrayOpExprHashTable;
struct JsonConstructorExprState;
+struct JsonbValue;
+struct JsonExprState;
/* Bits in ExprState->flags (see also execnodes.h for public flag bits): */
/* expression's interpreter has been initialized */
@@ -239,6 +241,11 @@ typedef enum ExprEvalOp
EEOP_XMLEXPR,
EEOP_JSON_CONSTRUCTOR,
EEOP_IS_JSON,
+ EEOP_JSONEXPR_SKIP,
+ EEOP_JSONEXPR_PATH,
+ EEOP_JSONEXPR_BEHAVIOR,
+ EEOP_JSONEXPR_COERCION,
+ EEOP_JSONEXPR_COERCION_FINISH,
EEOP_AGGREF,
EEOP_GROUPING_FUNC,
EEOP_WINDOW_FUNC,
@@ -692,6 +699,57 @@ typedef struct ExprEvalStep
JsonIsPredicate *pred; /* original expression node */
} is_json;
+ /* for EEOP_JSONEXPR_PATH */
+ struct
+ {
+ struct JsonExprState *jsestate;
+ } jsonexpr;
+
+ /* for EEOP_JSONEXPR_SKIP */
+ struct
+ {
+ /* Same as jsonexpr.jsestate */
+ struct JsonExprState *jsestate;
+
+ /* See ExecEvalJsonExprSkip() */
+ int jump_coercion;
+ int jump_passing_args;
+ } jsonexpr_skip;
+
+ /* for EEOP_JSONEXPR_BEHAVIOR */
+ struct
+ {
+ /* Same as jsonexpr.jsestate */
+ struct JsonExprState *jsestate;
+
+ /* See ExecEvalJsonExprBehavior() */
+ int jump_onerror_expr;
+ int jump_onempty_expr;
+ int jump_coercion;
+ int jump_skip_coercion;
+ } jsonexpr_behavior;
+
+ /* for EEOP_JSONEXPR_COERCION */
+ struct
+ {
+ /* Same as jsonexpr.jsestate */
+ struct JsonExprState *jsestate;
+
+ /* See ExecEvalJsonExprCoercion() */
+ int jump_coercion_error;
+ int jump_coercion_done;
+ } jsonexpr_coercion;
+
+ /* for EEOP_JSONEXPR_COERCION_FINISH */
+ struct
+ {
+ /* Same as jsonexpr.jsestate */
+ struct JsonExprState *jsestate;
+
+ /* See ExecEvalJsonExprCoercion() */
+ int jump_coercion_error;
+ int jump_coercion_done;
+ } jsonexpr_coercion_finish;
} d;
} ExprEvalStep;
@@ -755,6 +813,85 @@ typedef struct JsonConstructorExprState
int nargs;
} JsonConstructorExprState;
+/*
+ * Information computed before evaluating EEOP_JSONEXPR_PATH step.
+ */
+typedef struct JsonExprPreEvalState
+{
+ /* value/isnull for JsonExpr.formatted_expr */
+ NullableDatum formatted_expr;
+
+ /* value/isnull for JsonExpr.pathspec */
+ NullableDatum pathspec;
+
+ /* JsonPathVariable entries for JsonExpr.passing_values */
+ List *args;
+} JsonExprPreEvalState;
+
+/*
+ * State for evaluating a given JsonCoercion.
+ */
+typedef struct JsonCoercionState
+{
+ /* Expression used to evaluate the coercion */
+ JsonCoercion *coercion;
+
+ /* ExprEvalStep to compute this coercion's expression */
+ int jump_eval_expr;
+
+ /* For passing to EEOP_IOCOERCE that might be present in the expression */
+ ErrorSaveContext escontext;
+} JsonCoercionState;
+
+/*
+ * Information needed by EEOP_JSONEXPR_BEHAVIOR and EEOP_JSONEXPR_COERCION
+ * steps.
+ */
+typedef struct JsonExprPostEvalState
+{
+ /* Is JSON item empty? */
+ bool empty;
+
+ /* Did JSON item evaluation cause an error? */
+ bool error;
+
+ /* Cache for json_populate_type() called for coercion in some cases */
+ void *cache;
+
+ /*
+ * State for coercing the result of ExecEvalJsonExpr to the desired target
+ * type. 'jcstate' either points to JsonExprState.result_coercion or one
+ * of the entries in JsonExprState.item_jcstates chosen by
+ * ExecPrepareJsonItemCoercion() in the case of JSON_VALUE.
+ */
+ JsonCoercionState *jcstate;
+ bool coercing_behavior_expr; /* a hack for JSON_QUERY_OP */
+ bool coercion_error; /* error when coercing */
+ bool coercion_done;
+} JsonExprPostEvalState;
+
+/* State for evaluating a JsonExpr, too big to inline */
+typedef struct JsonExprState
+{
+ /* original expression node */
+ JsonExpr *jsexpr;
+
+ JsonExprPreEvalState pre_eval;
+ JsonExprPostEvalState post_eval;
+
+ struct
+ {
+ FmgrInfo *finfo; /* typinput function for output type */
+ Oid typioparam;
+ } input; /* I/O info for output type */
+
+ /*
+ * ExecEvalJsonExprCoercion() chooses either result_jcstate or one from
+ * item_jcstates to apply coercion to the final result if needed.
+ */
+ JsonCoercionState *result_jcstate;
+ List *item_jcstates; /* List of JsonCoercionState */
+} JsonExprState;
/* functions in execExpr.c */
extern void ExprEvalPushStep(ExprState *es, const ExprEvalStep *s);
@@ -808,6 +945,14 @@ extern void ExecEvalXmlExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalJsonConstructor(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op);
+extern int ExecEvalJsonExprSkip(ExprState *state, ExprEvalStep *op);
+extern int ExecEvalJsonExprBehavior(ExprState *state, ExprEvalStep *op);
+extern int ExecEvalJsonExprCoercion(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext,
+ Datum res, bool resnull);
+extern int ExecEvalJsonExprCoercionFinish(ExprState *state, ExprEvalStep *op);
+extern void ExecEvalJsonExpr(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
extern void ExecEvalGroupingFunc(ExprState *state, ExprEvalStep *op);
extern void ExecEvalSubPlan(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
diff --git a/src/include/fmgr.h b/src/include/fmgr.h
index b120f5e7fe..9e718479f9 100644
--- a/src/include/fmgr.h
+++ b/src/include/fmgr.h
@@ -85,6 +85,7 @@ typedef struct FmgrInfo
typedef struct FunctionCallInfoBaseData
{
FmgrInfo *flinfo; /* ptr to lookup info used for this call */
+#define FIELDNO_FUNCTIONCALLINFODATA_CONTEXT 1
fmNodePtr context; /* pass info about context of call */
fmNodePtr resultinfo; /* pass or return extra info about result */
Oid fncollation; /* collation for function to use */
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 3180703005..a850a1928b 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -112,6 +112,7 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
int location);
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
+extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index f637937cd2..c1f9ef844e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1716,6 +1716,23 @@ typedef struct TriggerTransition
/* Nodes for SQL/JSON support */
+/*
+ * JsonQuotes -
+ * representation of [KEEP|OMIT] QUOTES clause for JSON_QUERY()
+ */
+typedef enum JsonQuotes
+{
+ JS_QUOTES_UNSPEC, /* unspecified */
+ JS_QUOTES_KEEP, /* KEEP QUOTES */
+ JS_QUOTES_OMIT /* OMIT QUOTES */
+} JsonQuotes;
+
+/*
+ * JsonPathSpec -
+ * representation of JSON path constant
+ */
+typedef char *JsonPathSpec;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1727,6 +1744,48 @@ typedef struct JsonOutput
JsonReturning *returning; /* RETURNING FORMAT clause and type Oids */
} JsonOutput;
+/*
+ * JsonArgument -
+ * representation of argument from JSON PASSING clause
+ */
+typedef struct JsonArgument
+{
+ NodeTag type;
+ JsonValueExpr *val; /* argument value expression */
+ char *name; /* argument name */
+} JsonArgument;
+
+/*
+ * JsonCommon -
+ * representation of common syntax of functions using JSON path
+ */
+typedef struct JsonCommon
+{
+ NodeTag type;
+ JsonValueExpr *expr; /* context item expression */
+ Node *pathspec; /* JSON path specification expression */
+ char *pathname; /* path name, if any */
+ List *passing; /* list of PASSING clause arguments, if any */
+ int location; /* token location, or -1 if unknown */
+} JsonCommon;
+
+/*
+ * JsonFuncExpr -
+ * untransformed representation of JSON function expressions
+ */
+typedef struct JsonFuncExpr
+{
+ NodeTag type;
+ JsonExprOp op; /* expression type */
+ JsonCommon *common; /* common syntax */
+ JsonOutput *output; /* output clause, if specified */
+ JsonBehavior *on_empty; /* ON EMPTY behavior, if specified */
+ JsonBehavior *on_error; /* ON ERROR behavior, if specified */
+ JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
+ JsonQuotes quotes; /* omit or keep quotes? (JSON_QUERY only) */
+ int location; /* token location, or -1 if unknown */
+} JsonFuncExpr;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 60d72a876b..8f3723ef4c 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -1544,6 +1544,17 @@ typedef struct XmlExpr
int location;
} XmlExpr;
+/*
+ * JsonExprOp -
+ * enumeration of JSON functions using JSON path
+ */
+typedef enum JsonExprOp
+{
+ JSON_VALUE_OP, /* JSON_VALUE() */
+ JSON_QUERY_OP, /* JSON_QUERY() */
+ JSON_EXISTS_OP /* JSON_EXISTS() */
+} JsonExprOp;
+
/*
* JsonEncoding -
* representation of JSON ENCODING clause
@@ -1568,6 +1579,37 @@ typedef enum JsonFormatType
* jsonb */
} JsonFormatType;
+/*
+ * JsonBehaviorType -
+ * enumeration of behavior types used in JSON ON ... BEHAVIOR clause
+ *
+ * If enum members are reordered, get_json_behavior() from ruleutils.c
+ * must be updated accordingly.
+ */
+typedef enum JsonBehaviorType
+{
+ JSON_BEHAVIOR_NULL = 0,
+ JSON_BEHAVIOR_ERROR,
+ JSON_BEHAVIOR_EMPTY,
+ JSON_BEHAVIOR_TRUE,
+ JSON_BEHAVIOR_FALSE,
+ JSON_BEHAVIOR_UNKNOWN,
+ JSON_BEHAVIOR_EMPTY_ARRAY,
+ JSON_BEHAVIOR_EMPTY_OBJECT,
+ JSON_BEHAVIOR_DEFAULT
+} JsonBehaviorType;
+
+/*
+ * JsonWrapper -
+ * representation of WRAPPER clause for JSON_QUERY()
+ */
+typedef enum JsonWrapper
+{
+ JSW_NONE,
+ JSW_CONDITIONAL,
+ JSW_UNCONDITIONAL,
+} JsonWrapper;
+
/*
* JsonFormat -
* representation of JSON FORMAT clause
@@ -1662,6 +1704,79 @@ typedef struct JsonIsPredicate
int location; /* token location, or -1 if unknown */
} JsonIsPredicate;
+/*
+ * JsonBehavior -
+ * representation of JSON ON ERROR / EMPTY clause
+ */
+typedef struct JsonBehavior
+{
+ NodeTag type;
+ JsonBehaviorType btype; /* behavior type */
+ Node *default_expr; /* default expression, if any */
+ int location; /* token location, or -1 if unknown */
+} JsonBehavior;
+
+/*
+ * JsonCoercion -
+ * coercion from SQL/JSON item types to SQL types
+ */
+typedef struct JsonCoercion
+{
+ NodeTag type;
+ Node *expr; /* resulting expression coerced to target type */
+ bool via_populate; /* coerce result using json_populate_type()? */
+ bool via_io; /* coerce result using type input function? */
+ Oid collation; /* collation for coercion via I/O or populate */
+} JsonCoercion;
+
+/*
+ * JsonItemType
+ * Represents type codes to identify a JsonCoercion node to use when
+ * coercing a given SQL/JSON items to the output SQL type
+ *
+ * The comment next to each item type mentions the JsonbValue.jbvType of the
+ * source JsonbValue value to be coerced using the expression in the
+ * JsonCoercion node.
+ *
+ * Also, see InitJsonItemCoercions() and ExecPrepareJsonItemCoercion().
+ */
+typedef enum JsonItemType
+{
+ JsonItemTypeNull = 0, /* jbvNull */
+ JsonItemTypeString = 1, /* jbvString */
+ JsonItemTypeNumeric = 2, /* jbvNumeric */
+ JsonItemTypeBoolean = 3, /* jbvBool */
+ JsonItemTypeDate = 4, /* jbvDatetime: DATEOID */
+ JsonItemTypeTime = 5, /* jbvDatetime: TIMEOID */
+ JsonItemTypeTimetz = 6, /* jbvDatetime: TIMETZOID */
+ JsonItemTypeTimestamp = 7, /* jbvDatetime: TIMESTAMPOID */
+ JsonItemTypeTimestamptz = 8, /* jbvDatetime: TIMESTAMPTZOID */
+ JsonItemTypeComposite = 9 /* jbvArray, jbvObject, jbvBinary */
+} JsonItemType;
+
+/*
+ * JsonExpr -
+ * transformed representation of JSON_VALUE(), JSON_QUERY(), JSON_EXISTS()
+ */
+typedef struct JsonExpr
+{
+ Expr xpr;
+ JsonExprOp op; /* json function ID */
+ Node *formatted_expr; /* formatted context item expression */
+ JsonCoercion *result_coercion; /* resulting coercion to RETURNING type */
+ JsonFormat *format; /* context item format (JSON/JSONB) */
+ Node *path_spec; /* JSON path specification expression */
+ List *passing_names; /* PASSING argument names */
+ List *passing_values; /* PASSING argument values */
+ JsonReturning *returning; /* RETURNING clause type/format info */
+ JsonBehavior *on_empty; /* ON EMPTY behavior */
+ JsonBehavior *on_error; /* ON ERROR behavior */
+ List *item_coercions; /* coercions for JSON_VALUE */
+ JsonWrapper wrapper; /* WRAPPER for JSON_QUERY */
+ bool omit_quotes; /* KEEP/OMIT QUOTES for JSON_QUERY */
+ int location; /* token location, or -1 if unknown */
+} JsonExpr;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..0954d9fc7b 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -93,6 +93,7 @@ PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("conditional", CONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -147,11 +148,13 @@ PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("else", ELSE, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("empty", EMPTY_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("end", END_P, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("error", ERROR_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD, AS_LABEL)
@@ -233,10 +236,14 @@ PG_KEYWORD("join", JOIN, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json", JSON, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_array", JSON_ARRAY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_arrayagg", JSON_ARRAYAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_exists", JSON_EXISTS, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_object", JSON_OBJECT, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("keys", KEYS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -302,6 +309,7 @@ PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("omit", OMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("on", ON, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("only", ONLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -344,6 +352,7 @@ PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("quotes", QUOTES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("read", READ, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("real", REAL, COL_NAME_KEYWORD, BARE_LABEL)
@@ -414,6 +423,7 @@ PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
@@ -449,6 +459,7 @@ PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unbounded", UNBOUNDED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("unconditional", UNCONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("union", UNION, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("unique", UNIQUE, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/utils/formatting.h b/src/include/utils/formatting.h
index 0cad3a2709..9a9ae9754e 100644
--- a/src/include/utils/formatting.h
+++ b/src/include/utils/formatting.h
@@ -29,5 +29,6 @@ extern char *asc_initcap(const char *buff, size_t nbytes);
extern Datum parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
Oid *typid, int32 *typmod, int *tz,
struct Node *escontext);
+extern bool datetime_format_has_tz(const char *fmt_str);
#endif
diff --git a/src/include/utils/jsonb.h b/src/include/utils/jsonb.h
index 649a1644f2..ac279ee535 100644
--- a/src/include/utils/jsonb.h
+++ b/src/include/utils/jsonb.h
@@ -422,6 +422,9 @@ extern char *JsonbToCString(StringInfo out, JsonbContainer *in,
int estimated_len);
extern char *JsonbToCStringIndent(StringInfo out, JsonbContainer *in,
int estimated_len);
+extern Jsonb *JsonbMakeEmptyArray(void);
+extern Jsonb *JsonbMakeEmptyObject(void);
+extern char *JsonbUnquote(Jsonb *jb);
extern bool JsonbExtractScalar(JsonbContainer *jbc, JsonbValue *res);
extern const char *JsonbTypeName(JsonbValue *val);
diff --git a/src/include/utils/jsonfuncs.h b/src/include/utils/jsonfuncs.h
index c677ac8ff7..ab543b9423 100644
--- a/src/include/utils/jsonfuncs.h
+++ b/src/include/utils/jsonfuncs.h
@@ -15,6 +15,7 @@
#define JSONFUNCS_H
#include "common/jsonapi.h"
+#include "nodes/nodes.h"
#include "utils/jsonb.h"
/*
@@ -87,5 +88,9 @@ extern Datum datum_to_json(Datum val, JsonTypeCategory tcategory,
extern Datum datum_to_jsonb(Datum val, JsonTypeCategory tcategory,
Oid outfuncoid);
extern Datum jsonb_from_text(text *js, bool unique_keys);
+extern Datum json_populate_type(Datum json_val, Oid json_type,
+ Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext);
#endif
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index f0181e045f..5a37133847 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -16,7 +16,9 @@
#include "fmgr.h"
#include "nodes/pg_list.h"
+#include "nodes/primnodes.h"
#include "utils/jsonb.h"
+#include "utils/jsonfuncs.h"
typedef struct
{
@@ -184,6 +186,7 @@ extern bool jspGetBool(JsonPathItem *v);
extern char *jspGetString(JsonPathItem *v, int32 *len);
extern bool jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from,
JsonPathItem *to, int i);
+extern bool jspIsMutable(JsonPath *path, List *varnames, List *varexprs);
extern const char *jspOperationName(JsonPathItemType type);
@@ -261,4 +264,28 @@ extern bool jspConvertRegexFlags(uint32 xflags, int *result,
struct Node *escontext);
+/*
+ * Evaluation of jsonpath
+ */
+
+/* External variable passed into jsonpath. */
+typedef struct JsonPathVariable
+{
+ char *name;
+ Oid typid;
+ int32 typmod;
+ Datum value;
+ bool isnull;
+} JsonPathVariable;
+
+/* SQL/JSON item */
+extern void JsonItemFromDatum(Datum val, Oid typid, int32 typmod,
+ JsonbValue *res);
+
+extern bool JsonPathExists(Datum jb, JsonPath *path, bool *error, List *vars);
+extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
+ bool *empty, bool *error, List *vars);
+extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
+ bool *error, List *vars);
+
#endif
diff --git a/src/interfaces/ecpg/preproc/ecpg.trailer b/src/interfaces/ecpg/preproc/ecpg.trailer
index 435c139ec2..b2aa44f36d 100644
--- a/src/interfaces/ecpg/preproc/ecpg.trailer
+++ b/src/interfaces/ecpg/preproc/ecpg.trailer
@@ -651,6 +651,34 @@ var_type: simple_type
$$.type_index = mm_strdup("-1");
$$.type_sizeof = NULL;
}
+ | STRING_P
+ {
+ if (INFORMIX_MODE)
+ {
+ /* In Informix mode, "string" is automatically a typedef */
+ $$.type_enum = ECPGt_string;
+ $$.type_str = mm_strdup("char");
+ $$.type_dimension = mm_strdup("-1");
+ $$.type_index = mm_strdup("-1");
+ $$.type_sizeof = NULL;
+ }
+ else
+ {
+ /* Otherwise, legal only if user typedef'ed it */
+ struct typedefs *this = get_typedef("string", false);
+
+ $$.type_str = (this->type->type_enum == ECPGt_varchar || this->type->type_enum == ECPGt_bytea) ? EMPTY : mm_strdup(this->name);
+ $$.type_enum = this->type->type_enum;
+ $$.type_dimension = this->type->type_dimension;
+ $$.type_index = this->type->type_index;
+ if (this->type->type_sizeof && strlen(this->type->type_sizeof) != 0)
+ $$.type_sizeof = this->type->type_sizeof;
+ else
+ $$.type_sizeof = cat_str(3, mm_strdup("sizeof("), mm_strdup(this->name), mm_strdup(")"));
+
+ struct_member_list[struct_level] = ECPGstruct_member_dup(this->struct_member_list);
+ }
+ }
| INTERVAL ecpg_interval
{
$$.type_enum = ECPGt_interval;
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
new file mode 100644
index 0000000000..8b87580752
--- /dev/null
+++ b/src/test/regress/expected/json_sqljson.out
@@ -0,0 +1,18 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ERROR: JSON_EXISTS() is not yet implemented for the json type
+LINE 1: SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ERROR: JSON_VALUE() is not yet implemented for the json type
+LINE 1: SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_QUERY
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ERROR: JSON_QUERY() is not yet implemented for the json type
+LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
new file mode 100644
index 0000000000..8dcdb90fc8
--- /dev/null
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -0,0 +1,1071 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+ json_exists
+-------------
+
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb 'null', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+ json_exists
+-------------
+ f
+(1 row)
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+ json_exists
+-------------
+ t
+(1 row)
+
+-- extension: RETURNING clause
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING bool);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING bool);
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING int);
+ json_exists
+-------------
+ 1
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING int);
+ json_exists
+-------------
+ 0
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING text);
+ json_exists
+-------------
+ true
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING text);
+ json_exists
+-------------
+ false
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $[1]' RETURNING text FALSE ON ERROR);
+ json_exists
+-------------
+ false
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+ERROR: cannot cast type boolean to jsonb
+LINE 1: SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+ ^
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+ERROR: cannot cast type boolean to real
+LINE 1: SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+ ^
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSON); -- RETURNING FORMAT not allowed
+ERROR: cannot specify FORMAT in RETURNING clause of JSON_EXISTS()
+LINE 1: ...CT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSO...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING record);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL::jsonb, '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+ json_value
+------------
+ true
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+ json_value
+------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+ json_value
+------------
+ 123
+(1 row)
+
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+ERROR: SQL/JSON item cannot be cast to target type
+SELECT JSON_VALUE(jsonb '1.23', '$');
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "1.23"
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+ json_value
+------------
+ aa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "aaa"
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+ json_value
+------------
+ 111
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+ ?column?
+------------
+ 03-01-2017
+(1 row)
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null NULL ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[]', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '{}', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '1', '$.a');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+ json_value
+------------
+ error
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+ json_value
+------------
+ 3
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+ json_value
+------------
+ 0
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: " "
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 5
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+ERROR: cannot specify FORMAT in RETURNING clause of JSON_VALUE()
+LINE 1: ...CT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSO...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+ x | y
+---+----
+ 0 | -2
+ 1 | 2
+ 2 | -1
+(3 rows)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+-- Test timestamptz passing and output
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+ json_value
+--------------------------
+ Tue Feb 20 18:34:56 2018
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- JSON_QUERY
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+ json_query | json_query | json_query | json_query | json_query
+--------------------+--------------------+--------------------+----------------------+----------------------
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, null, "2"] | [1, null, "2"] | [1, null, "2"] | [[1, null, "2"]] | [[1, null, "2"]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+(6 rows)
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+ unspec | without | with cond | with uncond | with
+--------------------+--------------------+---------------------+----------------------+----------------------
+ | | | |
+ | | | |
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, 2, 3] | [1, 2, 3] | [1, 2, 3] | [[1, 2, 3]] | [[1, 2, 3]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+ | | [1, "2", null, [3]] | [1, "2", null, [3]] | [1, "2", null, [3]]
+(9 rows)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+ json_query
+------------
+ \x616161
+(1 row)
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTE...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTE...
+ ^
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_QUERY should return singleton item without wrapper
+HINT: Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+ json_query
+------------
+ [1,
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ \x7b7d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ \x7b7d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+ERROR: expected JSON array
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+ x | y | list
+---+---+--------------
+ 0 | 0 | []
+ 0 | 1 | [1]
+ 0 | 2 | [1, 2]
+ 0 | 3 | [1, 2, 3]
+ 0 | 4 | [1, 2, 3, 4]
+ 1 | 0 | []
+ 1 | 1 | [1]
+ 1 | 2 | [1, 2]
+ 1 | 3 | [1, 2, 3]
+ 1 | 4 | [1, 2, 3, 4]
+ 2 | 0 | []
+ 2 | 1 | []
+ 2 | 2 | [2]
+ 2 | 3 | [2, 3]
+ 2 | 4 | [2, 3, 4]
+ 3 | 0 | []
+ 3 | 1 | []
+ 3 | 2 | []
+ 3 | 3 | [3]
+ 3 | 4 | [3, 4]
+ 4 | 0 | []
+ 4 | 1 | []
+ 4 | 2 | []
+ 4 | 3 | []
+ 4 | 4 | [4]
+(25 rows)
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+-----------------------------------------------------
+ (1,aaa,"[1, ""2"", {}]","{""x"": [1, ""2"", {}]}",)
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+ unnest
+------------------------
+ {"a": 1, "b": ["foo"]}
+ {"a": 2, "c": {}}
+ 123
+(3 rows)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+--------------
+ {1,2,NULL,3}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+ json_query
+------------
+ 1
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- Test constraints
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT ('12' || i)::int ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+ CONSTRAINT test_jsonb_constraint6
+ CHECK (JSON_EXISTS(js::jsonb, 'strict $.a' RETURNING int TRUE ON ERROR) < 2)
+);
+\d test_jsonb_constraints
+ Table "public.test_jsonb_constraints"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+--------------------------------------------------------------------------------
+ js | text | | |
+ i | integer | | |
+ x | jsonb | | | JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+Check constraints:
+ "test_jsonb_constraint1" CHECK (js IS JSON)
+ "test_jsonb_constraint2" CHECK (JSON_EXISTS(js::jsonb, '$."a"' PASSING i + 5 AS int, i::text AS txt, ARRAY[1, 2, 3] AS arr))
+ "test_jsonb_constraint3" CHECK (JSON_VALUE(js::jsonb, '$."a"' RETURNING integer DEFAULT ('12'::text || i)::integer ON EMPTY ERROR ON ERROR) > i)
+ "test_jsonb_constraint4" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ "test_jsonb_constraint5" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+ "test_jsonb_constraint6" CHECK (JSON_EXISTS(js::jsonb, 'strict $."a"' RETURNING integer TRUE ON ERROR) < 2)
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+ check_clause
+------------------------------------------------------------------------------------------------------------------------
+ (JSON_EXISTS((js)::jsonb, 'strict $."a"' RETURNING integer TRUE ON ERROR) < 2)
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ (JSON_VALUE((js)::jsonb, '$."a"' RETURNING integer DEFAULT (('12'::text || i))::integer ON EMPTY ERROR ON ERROR) > i)
+ (js IS JSON)
+ JSON_EXISTS((js)::jsonb, '$."a"' PASSING (i + 5) AS int, (i)::text AS txt, ARRAY[1, 2, 3] AS arr)
+(6 rows)
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint1"
+DETAIL: Failing row contains (, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains (1, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ([], null, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ({"b": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint3"
+DETAIL: Failing row contains ({"a": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint5"
+DETAIL: Failing row contains ({"a": 7}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint4"
+DETAIL: Failing row contains ({"a": 10}, 1, [1, 2]).
+DROP TABLE test_jsonb_constraints;
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 4df9d8503b..9b0ecf049d 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -103,7 +103,7 @@ test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combo
# ----------
# Another group of parallel tests (JSON related)
# ----------
-test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson
+test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson json_sqljson jsonb_sqljson
# ----------
# Another group of parallel tests
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
new file mode 100644
index 0000000000..4f30fa46b9
--- /dev/null
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -0,0 +1,11 @@
+-- JSON_EXISTS
+
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+
+-- JSON_QUERY
+
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
new file mode 100644
index 0000000000..6c3f8c7e43
--- /dev/null
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -0,0 +1,341 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+SELECT JSON_EXISTS(jsonb 'null', '$');
+SELECT JSON_EXISTS(jsonb '[]', '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+
+-- extension: RETURNING clause
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING bool);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING bool);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING int);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING int);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING text);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING text);
+SELECT JSON_EXISTS(jsonb '1', 'strict $[1]' RETURNING text FALSE ON ERROR);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSON); -- RETURNING FORMAT not allowed
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING record);
+
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL::jsonb, '$');
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+
+SELECT JSON_VALUE(jsonb '123', '$');
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1.23', '$');
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null NULL ON ERROR);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON ERROR);
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+
+SELECT JSON_VALUE(jsonb '[]', '$');
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '{}', '$');
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1', '$.a');
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+
+-- Test timestamptz passing and output
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- JSON_QUERY
+
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- Test constraints
+
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT ('12' || i)::int ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+ CONSTRAINT test_jsonb_constraint6
+ CHECK (JSON_EXISTS(js::jsonb, 'strict $.a' RETURNING int TRUE ON ERROR) < 2)
+);
+
+\d test_jsonb_constraints
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+
+DROP TABLE test_jsonb_constraints;
+
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 8de90c4958..6f45542836 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1241,6 +1241,7 @@ Join
JoinCostWorkspace
JoinDomain
JoinExpr
+JsonFuncExpr
JoinHashEntry
JoinPath
JoinPathExtraData
@@ -1251,18 +1252,30 @@ JsObject
JsValue
JsonAggConstructor
JsonAggState
+JsonArgument
JsonArrayAgg
JsonArrayConstructor
JsonArrayQueryConstructor
JsonBaseObjectInfo
+JsonBehavior
+JsonBehaviorType
+JsonCoercion
+JsonCoercionState
+JsonCommon
JsonConstructorExpr
JsonConstructorExprState
JsonConstructorType
JsonEncoding
+JsonExpr
+JsonExprOp
+JsonExprPostEvalState
+JsonExprPreEvalState
+JsonExprState
JsonFormat
JsonFormatType
JsonHashEntry
JsonIsPredicate
+JsonItemType
JsonIterateStringValuesAction
JsonKeyValue
JsonLexContext
@@ -1280,6 +1293,7 @@ JsonParseContext
JsonParseErrorType
JsonPath
JsonPathBool
+JsonPathDatatypeStatus
JsonPathExecContext
JsonPathExecResult
JsonPathGinAddPathItemFunc
@@ -1292,10 +1306,15 @@ JsonPathGinPathItem
JsonPathItem
JsonPathItemType
JsonPathKeyword
+JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
JsonPathString
+JsonPathVarCallback
+JsonPathVariable
+JsonPathVariableEvalContext
+JsonQuotes
JsonReturning
JsonScalarExpr
JsonSemAction
@@ -1312,6 +1331,7 @@ JsonValueExpr
JsonValueList
JsonValueListIterator
JsonValueType
+JsonWrapper
Jsonb
JsonbAggState
JsonbContainer
--
2.35.3
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-10-02 04:24 ` Amit Langote <[email protected]>
2023-10-02 05:26 ` Re: remaining sql/json patches Amit Langote <[email protected]>
1 sibling, 1 reply; 94+ messages in thread
From: Amit Langote @ 2023-10-02 04:24 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
On Fri, Sep 29, 2023 at 1:57 PM Amit Langote <[email protected]> wrote:
> On Thu, Sep 28, 2023 at 8:04 PM Alvaro Herrera <[email protected]> wrote:
> > On 2023-Sep-27, Amit Langote wrote:
> > > Maybe the following is better:
> > >
> > > + /*
> > > + * For expression nodes that support soft errors. Should be set to NULL
> > > + * before calling ExecInitExprRec() if the caller wants errors thrown.
> > > + */
> > >
> > > ...as in the attached.
> >
> > That's good.
> >
> > > Alvaro, do you think your concern regarding escontext not being in the
> > > right spot in the ExprState struct is addressed? It doesn't seem very
> > > critical to me to place it in the struct's 1st cacheline, because
> > > escontext is not accessed in performance critical paths such as during
> > > expression evaluation, especially with the latest version. (It would
> > > get accessed during evaluation with previous versions.)
> > >
> > > If so, I'd like to move ahead with committing it.
> >
> > Yeah, looks OK to me in v21.
>
> Thanks. I will push the attached 0001 shortly.
Pushed this 30 min ago (no email on -committers yet!) and am looking
at the following llvm crash reported by buildfarm animal pogona [1]:
#0 __pthread_kill_implementation (threadid=<optimized out>,
signo=signo@entry=6, no_tid=no_tid@entry=0) at
./nptl/pthread_kill.c:44
44 ./nptl/pthread_kill.c: No such file or directory.
#0 __pthread_kill_implementation (threadid=<optimized out>,
signo=signo@entry=6, no_tid=no_tid@entry=0) at
./nptl/pthread_kill.c:44
#1 0x00007f5bcebcb15f in __pthread_kill_internal (signo=6,
threadid=<optimized out>) at ./nptl/pthread_kill.c:78
#2 0x00007f5bceb7d472 in __GI_raise (sig=sig@entry=6) at
../sysdeps/posix/raise.c:26
#3 0x00007f5bceb674b2 in __GI_abort () at ./stdlib/abort.c:79
#4 0x00007f5bceb673d5 in __assert_fail_base (fmt=0x7f5bcecdbdc8
"%s%s%s:%u: %s%sAssertion `%s' failed.\\n%n",
assertion=assertion@entry=0x7f5bc1336419 "(i >= FTy->getNumParams() ||
FTy->getParamType(i) == Args[i]->getType()) && \\"Calling a function
with a bad signature!\\"", file=file@entry=0x7f5bc1336051
"/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp",
line=line@entry=299, function=function@entry=0x7f5bc13362af "void
llvm::CallInst::init(llvm::FunctionType *, llvm::Value *,
ArrayRef<llvm::Value *>, ArrayRef<llvm::OperandBundleDef>, const
llvm::Twine &)") at ./assert/assert.c:92
#5 0x00007f5bceb763a2 in __assert_fail (assertion=0x7f5bc1336419 "(i
>= FTy->getNumParams() || FTy->getParamType(i) == Args[i]->getType())
&& \\"Calling a function with a bad signature!\\"",
file=0x7f5bc1336051
"/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp", line=299,
function=0x7f5bc13362af "void llvm::CallInst::init(llvm::FunctionType
*, llvm::Value *, ArrayRef<llvm::Value *>,
ArrayRef<llvm::OperandBundleDef>, const llvm::Twine &)") at
./assert/assert.c:101
#6 0x00007f5bc110f138 in llvm::CallInst::init (this=0x557a91f3e508,
FTy=0x557a91ed9ae0, Func=0x557a91f8be88, Args=..., Bundles=...,
NameStr=...) at
/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp:297
#7 0x00007f5bc0fa579d in llvm::CallInst::CallInst
(this=0x557a91f3e508, Ty=0x557a91ed9ae0, Func=0x557a91f8be88,
Args=..., Bundles=..., NameStr=..., InsertBefore=0x0) at
/home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1934
#8 0x00007f5bc0fa538c in llvm::CallInst::Create (Ty=0x557a91ed9ae0,
Func=0x557a91f8be88, Args=..., Bundles=..., NameStr=...,
InsertBefore=0x0) at
/home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1444
#9 0x00007f5bc0fa51f9 in llvm::IRBuilder<llvm::ConstantFolder,
llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
FTy=0x557a91ed9ae0, Callee=0x557a91f8be88, Args=..., Name=...,
FPMathTag=0x0) at
/home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1669
#10 0x00007f5bc100edda in llvm::IRBuilder<llvm::ConstantFolder,
llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
Callee=0x557a91f8be88, Args=..., Name=..., FPMathTag=0x0) at
/home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1663
#11 0x00007f5bc100714e in LLVMBuildCall (B=0x557a91f9c6a0,
Fn=0x557a91f8be88, Args=0x7ffde6fa0b50, NumArgs=6, Name=0x7f5bc30b648c
"funccall_iocoerce_in_safe") at
/home/bf/src/llvm-project-5/llvm/lib/IR/Core.cpp:2964
#12 0x00007f5bc30af861 in llvm_compile_expr (state=0x557a91fbeac0) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/jit/llvm/llvmjit_expr.c:1373
#13 0x0000557a915992db in jit_compile_expr
(state=state@entry=0x557a91fbeac0) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/jit/jit.c:177
#14 0x0000557a9123071d in ExecReadyExpr
(state=state@entry=0x557a91fbeac0) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execExpr.c:880
#15 0x0000557a912340d7 in ExecBuildProjectionInfo
(targetList=0x557a91fa6b58, econtext=<optimized out>, slot=<optimized
out>, parent=parent@entry=0x557a91f430a8,
inputDesc=inputDesc@entry=0x0) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execExpr.c:484
#16 0x0000557a9124e61e in ExecAssignProjectionInfo
(planstate=planstate@entry=0x557a91f430a8,
inputDesc=inputDesc@entry=0x0) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execUtils.c:547
#17 0x0000557a91274961 in ExecInitNestLoop
(node=node@entry=0x557a91f9e5d8, estate=estate@entry=0x557a91f425a0,
eflags=<optimized out>, eflags@entry=33) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/nodeNestloop.c:308
#18 0x0000557a9124760f in ExecInitNode (node=0x557a91f9e5d8,
estate=estate@entry=0x557a91f425a0, eflags=eflags@entry=33) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execProcnode.c:298
#19 0x0000557a91255d39 in ExecInitAgg (node=node@entry=0x557a91f91540,
estate=estate@entry=0x557a91f425a0, eflags=eflags@entry=33) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/nodeAgg.c:3306
#20 0x0000557a912476bf in ExecInitNode (node=0x557a91f91540,
estate=estate@entry=0x557a91f425a0, eflags=eflags@entry=33) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execProcnode.c:341
#21 0x0000557a912770c3 in ExecInitSort
(node=node@entry=0x557a91f9e850, estate=estate@entry=0x557a91f425a0,
eflags=eflags@entry=33) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/nodeSort.c:265
#22 0x0000557a91247667 in ExecInitNode
(node=node@entry=0x557a91f9e850, estate=estate@entry=0x557a91f425a0,
eflags=eflags@entry=33) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execProcnode.c:321
#23 0x0000557a912402f5 in InitPlan (eflags=33,
queryDesc=0x557a91fa6fb8) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execMain.c:968
#24 standard_ExecutorStart (queryDesc=queryDesc@entry=0x557a91fa6fb8,
eflags=33, eflags@entry=1) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execMain.c:266
#25 0x0000557a912403c9 in ExecutorStart
(queryDesc=queryDesc@entry=0x557a91fa6fb8, eflags=1) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/executor/execMain.c:145
#26 0x0000557a911c2153 in ExplainOnePlan
(plannedstmt=plannedstmt@entry=0x557a91fa6ea8, into=into@entry=0x0,
es=es@entry=0x557a91f932e8,
queryString=queryString@entry=0x557a91dbd650 "EXPLAIN (COSTS
OFF)\\nSELECT DISTINCT (i || '/' || j)::pg_lsn f\\n FROM
generate_series(1, 10) i,\\n generate_series(1, 10) j,\\n
generate_series(1, 5) k\\n WHERE i <= 10 AND j > 0 AND j <= 10\\n
O"..., params=params@entry=0x0, queryEnv=queryEnv@entry=0x0,
planduration=0x7ffde6fa1258, bufusage=0x0) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/commands/explain.c:590
#27 0x0000557a911c23b2 in ExplainOneQuery (query=<optimized out>,
cursorOptions=cursorOptions@entry=2048, into=into@entry=0x0,
es=es@entry=0x557a91f932e8, queryString=0x557a91dbd650 "EXPLAIN (COSTS
OFF)\\nSELECT DISTINCT (i || '/' || j)::pg_lsn f\\n FROM
generate_series(1, 10) i,\\n generate_series(1, 10) j,\\n
generate_series(1, 5) k\\n WHERE i <= 10 AND j > 0 AND j <= 10\\n
O"..., params=params@entry=0x0, queryEnv=0x0) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/commands/explain.c:419
#28 0x0000557a911c2ddb in ExplainQuery
(pstate=pstate@entry=0x557a91f3eb18, stmt=stmt@entry=0x557a91e881d0,
params=params@entry=0x0, dest=dest@entry=0x557a91f3ea88) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/include/nodes/nodes.h:193
#29 0x0000557a91413811 in standard_ProcessUtility
(pstmt=0x557a91e88280, queryString=0x557a91dbd650 "EXPLAIN (COSTS
OFF)\\nSELECT DISTINCT (i || '/' || j)::pg_lsn f\\n FROM
generate_series(1, 10) i,\\n generate_series(1, 10) j,\\n
generate_series(1, 5) k\\n WHERE i <= 10 AND j > 0 AND j <= 10\\n
O"..., readOnlyTree=<optimized out>, context=PROCESS_UTILITY_TOPLEVEL,
params=0x0, queryEnv=0x0, dest=0x557a91f3ea88, qc=0x7ffde6fa1500) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/tcop/utility.c:870
#30 0x0000557a91413ed9 in ProcessUtility
(pstmt=pstmt@entry=0x557a91e88280, queryString=<optimized out>,
readOnlyTree=<optimized out>,
context=context@entry=PROCESS_UTILITY_TOPLEVEL, params=<optimized
out>, queryEnv=<optimized out>, dest=0x557a91f3ea88,
qc=0x7ffde6fa1500) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/tcop/utility.c:530
#31 0x0000557a91411537 in PortalRunUtility
(portal=portal@entry=0x557a91e35970, pstmt=0x557a91e88280,
isTopLevel=true, setHoldSnapshot=setHoldSnapshot@entry=true,
dest=dest@entry=0x557a91f3ea88, qc=qc@entry=0x7ffde6fa1500) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/tcop/pquery.c:1158
#32 0x0000557a914119a4 in FillPortalStore
(portal=portal@entry=0x557a91e35970, isTopLevel=isTopLevel@entry=true)
at /home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/include/nodes/nodes.h:193
#33 0x0000557a91411d6d in PortalRun
(portal=portal@entry=0x557a91e35970,
count=count@entry=9223372036854775807,
isTopLevel=isTopLevel@entry=true, run_once=run_once@entry=true,
dest=dest@entry=0x557a91e88900, altdest=altdest@entry=0x557a91e88900,
qc=0x7ffde6fa1700) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/tcop/pquery.c:763
#34 0x0000557a9140d65f in exec_simple_query
(query_string=query_string@entry=0x557a91dbd650 "EXPLAIN (COSTS
OFF)\\nSELECT DISTINCT (i || '/' || j)::pg_lsn f\\n FROM
generate_series(1, 10) i,\\n generate_series(1, 10) j,\\n
generate_series(1, 5) k\\n WHERE i <= 10 AND j > 0 AND j <= 10\\n
O"...) at /home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/tcop/postgres.c:1272
#35 0x0000557a9140e305 in PostgresMain (dbname=<optimized out>,
username=<optimized out>) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/tcop/postgres.c:4652
#36 0x0000557a91372bf0 in BackendRun (port=0x557a91de8730) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/postmaster/postmaster.c:4439
#37 BackendStartup (port=0x557a91de8730) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/postmaster/postmaster.c:4167
#38 ServerLoop () at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/postmaster/postmaster.c:1781
#39 0x0000557a9137488e in PostmasterMain (argc=argc@entry=8,
argv=argv@entry=0x557a91d7cc10) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/postmaster/postmaster.c:1465
#40 0x0000557a912a001e in main (argc=8, argv=0x557a91d7cc10) at
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/main/main.c:198
$1 = {si_signo = 6, si_errno = 0, si_code = -6, _sifields = {_pad =
{3110875, 1000, 0 <repeats 26 times>}, _kill = {si_pid = 3110875,
si_uid = 1000}, _timer = {si_tid = 3110875, si_overrun = 1000,
si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _rt = {si_pid =
3110875, si_uid = 1000, si_sigval = {sival_int = 0, sival_ptr = 0x0}},
_sigchld = {si_pid = 3110875, si_uid = 1000, si_status = 0, si_utime =
0, si_stime = 0}, _sigfault = {si_addr = 0x3e8002f77db, _addr_lsb = 0,
_addr_bnd = {_lower = 0x0, _upper = 0x0}}, _sigpoll = {si_band =
4294970406875, si_fd = 0}, _sigsys = {_call_addr = 0x3e8002f77db,
_syscall = 0, _arch = 0}}}
This seems to me to be complaining about the following addition:
+ {
+ Oid ioparam = op->d.iocoerce.typioparam;
+ LLVMValueRef v_params[6];
+ LLVMValueRef v_success;
+
+ v_params[0] = l_ptr_const(op->d.iocoerce.finfo_in,
+ l_ptr(StructFmgrInfo));
+ v_params[1] = v_output;
+ v_params[2] = l_oid_const(lc, ioparam);
+ v_params[3] = l_int32_const(lc, -1);
+ v_params[4] = l_ptr_const(op->d.iocoerce.escontext,
+
l_ptr(StructErrorSaveContext));
- LLVMBuildStore(b, v_retval, v_resvaluep);
+ /*
+ * InputFunctionCallSafe() will write directly into
+ * *op->resvalue.
+ */
+ v_params[5] = v_resvaluep;
+
+ v_success = LLVMBuildCall(b, llvm_pg_func(mod,
"InputFunctionCallSafe"),
+ v_params, lengthof(v_params),
+ "funccall_iocoerce_in_safe");
+
+ /*
+ * Return null if InputFunctionCallSafe() encountered
+ * an error.
+ */
+ v_resnullp = LLVMBuildICmp(b, LLVMIntEQ, v_success,
+ l_sbool_const(0), "");
+ }
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
[1] https://buildfarm.postgresql.org/cgi-bin/show_log.pl?nm=pogona&dt=2023-10-02%2003%3A50%3A20
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-02 04:24 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-10-02 05:26 ` Amit Langote <[email protected]>
2023-10-03 13:11 ` Re: remaining sql/json patches Amit Langote <[email protected]>
0 siblings, 1 reply; 94+ messages in thread
From: Amit Langote @ 2023-10-02 05:26 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers; jian he <[email protected]>
On Mon, Oct 2, 2023 at 1:24 PM Amit Langote <[email protected]> wrote:
> Pushed this 30 min ago (no email on -committers yet!) and am looking
> at the following llvm crash reported by buildfarm animal pogona [1]:
>
> #4 0x00007f5bceb673d5 in __assert_fail_base (fmt=0x7f5bcecdbdc8
> "%s%s%s:%u: %s%sAssertion `%s' failed.\\n%n",
> assertion=assertion@entry=0x7f5bc1336419 "(i >= FTy->getNumParams() ||
> FTy->getParamType(i) == Args[i]->getType()) && \\"Calling a function
> with a bad signature!\\"", file=file@entry=0x7f5bc1336051
> "/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp",
> line=line@entry=299, function=function@entry=0x7f5bc13362af "void
> llvm::CallInst::init(llvm::FunctionType *, llvm::Value *,
> ArrayRef<llvm::Value *>, ArrayRef<llvm::OperandBundleDef>, const
> llvm::Twine &)") at ./assert/assert.c:92
> #5 0x00007f5bceb763a2 in __assert_fail (assertion=0x7f5bc1336419 "(i
> >= FTy->getNumParams() || FTy->getParamType(i) == Args[i]->getType())
> && \\"Calling a function with a bad signature!\\"",
> file=0x7f5bc1336051
> "/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp", line=299,
> function=0x7f5bc13362af "void llvm::CallInst::init(llvm::FunctionType
> *, llvm::Value *, ArrayRef<llvm::Value *>,
> ArrayRef<llvm::OperandBundleDef>, const llvm::Twine &)") at
> ./assert/assert.c:101
> #6 0x00007f5bc110f138 in llvm::CallInst::init (this=0x557a91f3e508,
> FTy=0x557a91ed9ae0, Func=0x557a91f8be88, Args=..., Bundles=...,
> NameStr=...) at
> /home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp:297
> #7 0x00007f5bc0fa579d in llvm::CallInst::CallInst
> (this=0x557a91f3e508, Ty=0x557a91ed9ae0, Func=0x557a91f8be88,
> Args=..., Bundles=..., NameStr=..., InsertBefore=0x0) at
> /home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1934
> #8 0x00007f5bc0fa538c in llvm::CallInst::Create (Ty=0x557a91ed9ae0,
> Func=0x557a91f8be88, Args=..., Bundles=..., NameStr=...,
> InsertBefore=0x0) at
> /home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1444
> #9 0x00007f5bc0fa51f9 in llvm::IRBuilder<llvm::ConstantFolder,
> llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
> FTy=0x557a91ed9ae0, Callee=0x557a91f8be88, Args=..., Name=...,
> FPMathTag=0x0) at
> /home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1669
> #10 0x00007f5bc100edda in llvm::IRBuilder<llvm::ConstantFolder,
> llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
> Callee=0x557a91f8be88, Args=..., Name=..., FPMathTag=0x0) at
> /home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1663
> #11 0x00007f5bc100714e in LLVMBuildCall (B=0x557a91f9c6a0,
> Fn=0x557a91f8be88, Args=0x7ffde6fa0b50, NumArgs=6, Name=0x7f5bc30b648c
> "funccall_iocoerce_in_safe") at
> /home/bf/src/llvm-project-5/llvm/lib/IR/Core.cpp:2964
> #12 0x00007f5bc30af861 in llvm_compile_expr (state=0x557a91fbeac0) at
>
/home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/jit/llvm/llvmjit_expr.c:1373
>
> This seems to me to be complaining about the following addition:
>
> + {
> + Oid ioparam = op->d.iocoerce.typioparam;
> + LLVMValueRef v_params[6];
> + LLVMValueRef v_success;
> +
> + v_params[0] = l_ptr_const(op->d.iocoerce.finfo_in,
> + l_ptr(StructFmgrInfo));
> + v_params[1] = v_output;
> + v_params[2] = l_oid_const(lc, ioparam);
> + v_params[3] = l_int32_const(lc, -1);
> + v_params[4] =
l_ptr_const(op->d.iocoerce.escontext,
> +
> l_ptr(StructErrorSaveContext));
>
> - LLVMBuildStore(b, v_retval, v_resvaluep);
> + /*
> + * InputFunctionCallSafe() will write directly
into
> + * *op->resvalue.
> + */
> + v_params[5] = v_resvaluep;
> +
> + v_success = LLVMBuildCall(b, llvm_pg_func(mod,
> "InputFunctionCallSafe"),
> + v_params,
lengthof(v_params),
> +
"funccall_iocoerce_in_safe");
> +
> + /*
> + * Return null if InputFunctionCallSafe()
encountered
> + * an error.
> + */
> + v_resnullp = LLVMBuildICmp(b, LLVMIntEQ,
v_success,
> + l_sbool_const(0), "");
> + }
Although most animals except pogona looked fine, I've decided to revert the
patch for now.
IIUC, LLVM is complaining that the code in the above block is not passing
the arguments of InputFunctionCallSafe() using the correct types. I'm not
exactly sure which particular argument is not handled correctly in the
above code, but perhaps it's:
+ v_params[1] = v_output;
which maps to char *str argument of InputFunctionCallSafe(). v_output is
set in the code preceding the above block as follows:
/* and call output function (can never return NULL) */
v_output = LLVMBuildCall(b, v_fn_out, &v_fcinfo_out,
1, "funccall_coerce_out");
I thought that it would be fine to pass it as-is to the call of
InputFunctionCallSafe() given that v_fn_out is a call to a function that
returns char *, but perhaps not.
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-02 04:24 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-02 05:26 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-10-03 13:11 ` Amit Langote <[email protected]>
2023-10-04 13:26 ` Re: remaining sql/json patches Amit Langote <[email protected]>
0 siblings, 1 reply; 94+ messages in thread
From: Amit Langote @ 2023-10-03 13:11 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers; jian he <[email protected]>
On Mon, Oct 2, 2023 at 2:26 PM Amit Langote <[email protected]> wrote:
> On Mon, Oct 2, 2023 at 1:24 PM Amit Langote <[email protected]> wrote:
> > Pushed this 30 min ago (no email on -committers yet!) and am looking
> > at the following llvm crash reported by buildfarm animal pogona [1]:
> >
> > #4 0x00007f5bceb673d5 in __assert_fail_base (fmt=0x7f5bcecdbdc8
> > "%s%s%s:%u: %s%sAssertion `%s' failed.\\n%n",
> > assertion=assertion@entry=0x7f5bc1336419 "(i >= FTy->getNumParams() ||
> > FTy->getParamType(i) == Args[i]->getType()) && \\"Calling a function
> > with a bad signature!\\"", file=file@entry=0x7f5bc1336051
> > "/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp",
> > line=line@entry=299, function=function@entry=0x7f5bc13362af "void
> > llvm::CallInst::init(llvm::FunctionType *, llvm::Value *,
> > ArrayRef<llvm::Value *>, ArrayRef<llvm::OperandBundleDef>, const
> > llvm::Twine &)") at ./assert/assert.c:92
> > #5 0x00007f5bceb763a2 in __assert_fail (assertion=0x7f5bc1336419 "(i
> > >= FTy->getNumParams() || FTy->getParamType(i) == Args[i]->getType())
> > && \\"Calling a function with a bad signature!\\"",
> > file=0x7f5bc1336051
> > "/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp", line=299,
> > function=0x7f5bc13362af "void llvm::CallInst::init(llvm::FunctionType
> > *, llvm::Value *, ArrayRef<llvm::Value *>,
> > ArrayRef<llvm::OperandBundleDef>, const llvm::Twine &)") at
> > ./assert/assert.c:101
> > #6 0x00007f5bc110f138 in llvm::CallInst::init (this=0x557a91f3e508,
> > FTy=0x557a91ed9ae0, Func=0x557a91f8be88, Args=..., Bundles=...,
> > NameStr=...) at
> > /home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp:297
> > #7 0x00007f5bc0fa579d in llvm::CallInst::CallInst
> > (this=0x557a91f3e508, Ty=0x557a91ed9ae0, Func=0x557a91f8be88,
> > Args=..., Bundles=..., NameStr=..., InsertBefore=0x0) at
> > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1934
> > #8 0x00007f5bc0fa538c in llvm::CallInst::Create (Ty=0x557a91ed9ae0,
> > Func=0x557a91f8be88, Args=..., Bundles=..., NameStr=...,
> > InsertBefore=0x0) at
> > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1444
> > #9 0x00007f5bc0fa51f9 in llvm::IRBuilder<llvm::ConstantFolder,
> > llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
> > FTy=0x557a91ed9ae0, Callee=0x557a91f8be88, Args=..., Name=...,
> > FPMathTag=0x0) at
> > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1669
> > #10 0x00007f5bc100edda in llvm::IRBuilder<llvm::ConstantFolder,
> > llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
> > Callee=0x557a91f8be88, Args=..., Name=..., FPMathTag=0x0) at
> > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1663
> > #11 0x00007f5bc100714e in LLVMBuildCall (B=0x557a91f9c6a0,
> > Fn=0x557a91f8be88, Args=0x7ffde6fa0b50, NumArgs=6, Name=0x7f5bc30b648c
> > "funccall_iocoerce_in_safe") at
> > /home/bf/src/llvm-project-5/llvm/lib/IR/Core.cpp:2964
> > #12 0x00007f5bc30af861 in llvm_compile_expr (state=0x557a91fbeac0) at
> > /home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/jit/llvm/llvmjit_expr.c:1373
> >
> > This seems to me to be complaining about the following addition:
> >
> > + {
> > + Oid ioparam = op->d.iocoerce.typioparam;
> > + LLVMValueRef v_params[6];
> > + LLVMValueRef v_success;
> > +
> > + v_params[0] = l_ptr_const(op->d.iocoerce.finfo_in,
> > + l_ptr(StructFmgrInfo));
> > + v_params[1] = v_output;
> > + v_params[2] = l_oid_const(lc, ioparam);
> > + v_params[3] = l_int32_const(lc, -1);
> > + v_params[4] = l_ptr_const(op->d.iocoerce.escontext,
> > +
> > l_ptr(StructErrorSaveContext));
> >
> > - LLVMBuildStore(b, v_retval, v_resvaluep);
> > + /*
> > + * InputFunctionCallSafe() will write directly into
> > + * *op->resvalue.
> > + */
> > + v_params[5] = v_resvaluep;
> > +
> > + v_success = LLVMBuildCall(b, llvm_pg_func(mod,
> > "InputFunctionCallSafe"),
> > + v_params, lengthof(v_params),
> > + "funccall_iocoerce_in_safe");
> > +
> > + /*
> > + * Return null if InputFunctionCallSafe() encountered
> > + * an error.
> > + */
> > + v_resnullp = LLVMBuildICmp(b, LLVMIntEQ, v_success,
> > + l_sbool_const(0), "");
> > + }
>
> Although most animals except pogona looked fine, I've decided to revert the patch for now.
>
> IIUC, LLVM is complaining that the code in the above block is not passing the arguments of InputFunctionCallSafe() using the correct types. I'm not exactly sure which particular argument is not handled correctly in the above code, but perhaps it's:
>
>
> + v_params[1] = v_output;
>
> which maps to char *str argument of InputFunctionCallSafe(). v_output is set in the code preceding the above block as follows:
>
> /* and call output function (can never return NULL) */
> v_output = LLVMBuildCall(b, v_fn_out, &v_fcinfo_out,
> 1, "funccall_coerce_out");
>
> I thought that it would be fine to pass it as-is to the call of InputFunctionCallSafe() given that v_fn_out is a call to a function that returns char *, but perhaps not.
OK, I think I could use some help from LLVM experts here.
So, the LLVM code involving setting up a call to
InputFunctionCallSafe() seems to *work*, but BF animal pogona's debug
build (?) is complaining that the parameter types don't match up.
Parameters are set up as follows:
+ {
+ Oid ioparam = op->d.iocoerce.typioparam;
+ LLVMValueRef v_params[6];
+ LLVMValueRef v_success;
+
+ v_params[0] = l_ptr_const(op->d.iocoerce.finfo_in,
+ l_ptr(StructFmgrInfo));
+ v_params[1] = v_output;
+ v_params[2] = l_oid_const(lc, ioparam);
+ v_params[3] = l_int32_const(lc, -1);
+ v_params[4] = l_ptr_const(op->d.iocoerce.escontext,
+
l_ptr(StructErrorSaveContext));
+ /*
+ * InputFunctionCallSafe() will write directly into
+ * *op->resvalue.
+ */
+ v_params[5] = v_resvaluep;
+
+ v_success = LLVMBuildCall(b, llvm_pg_func(mod,
"InputFunctionCallSafe"),
+ v_params, lengthof(v_params),
+ "funccall_iocoerce_in_safe");
+
+ /*
+ * Return null if InputFunctionCallSafe() encountered
+ * an error.
+ */
+ v_resnullp = LLVMBuildICmp(b, LLVMIntEQ, v_success,
+ l_sbool_const(0), "");
+ }
And here's InputFunctionCallSafe's signature:
bool
InputFunctionCallSafe(FmgrInfo *flinfo, Datum d,
Oid typioparam, int32 typmod,
fmNodePtr escontext,
Datum *result)
I suspected that assignment to either param[1] or param[5] might be wrong.
param[1] in InputFunctionCallSafe's signature is char *, but the code
assigns it v_output, which is an LLVMValueRef for the output
function's output, a Datum, so I thought LLVM's type checker is
complaining that I'm trying to pass the Datum to char * without
appropriate conversion.
param[5] in InputFunctionCallSafe's signature is Node *, but the above
code is assigning it an LLVMValueRef for iocoerce's escontext whose
type is ErrorSaveContext.
Maybe some other param is wrong.
I tried various ways to fix both, but with no success. My way of
checking for failure is to disassemble the IR code in .bc files
(generated with jit_dump_bitcode) with llvm-dis and finding that it
gives me errors such as:
$ llvm-dis 58536.0.bc
llvm-dis: error: Invalid record (Producer: 'LLVM7.0.1' Reader: 'LLVM 7.0.1')
$ llvm-dis 58536.0.bc
llvm-dis: error: Invalid cast (Producer: 'LLVM7.0.1' Reader: 'LLVM 7.0.1')
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-02 04:24 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-02 05:26 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-03 13:11 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-10-04 13:26 ` Amit Langote <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Amit Langote @ 2023-10-04 13:26 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers; jian he <[email protected]>
On Tue, Oct 3, 2023 at 10:11 PM Amit Langote <[email protected]> wrote:
> On Mon, Oct 2, 2023 at 2:26 PM Amit Langote <[email protected]> wrote:
> > On Mon, Oct 2, 2023 at 1:24 PM Amit Langote <[email protected]> wrote:
> > > Pushed this 30 min ago (no email on -committers yet!) and am looking
> > > at the following llvm crash reported by buildfarm animal pogona [1]:
> > >
> > > #4 0x00007f5bceb673d5 in __assert_fail_base (fmt=0x7f5bcecdbdc8
> > > "%s%s%s:%u: %s%sAssertion `%s' failed.\\n%n",
> > > assertion=assertion@entry=0x7f5bc1336419 "(i >= FTy->getNumParams() ||
> > > FTy->getParamType(i) == Args[i]->getType()) && \\"Calling a function
> > > with a bad signature!\\"", file=file@entry=0x7f5bc1336051
> > > "/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp",
> > > line=line@entry=299, function=function@entry=0x7f5bc13362af "void
> > > llvm::CallInst::init(llvm::FunctionType *, llvm::Value *,
> > > ArrayRef<llvm::Value *>, ArrayRef<llvm::OperandBundleDef>, const
> > > llvm::Twine &)") at ./assert/assert.c:92
> > > #5 0x00007f5bceb763a2 in __assert_fail (assertion=0x7f5bc1336419 "(i
> > > >= FTy->getNumParams() || FTy->getParamType(i) == Args[i]->getType())
> > > && \\"Calling a function with a bad signature!\\"",
> > > file=0x7f5bc1336051
> > > "/home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp", line=299,
> > > function=0x7f5bc13362af "void llvm::CallInst::init(llvm::FunctionType
> > > *, llvm::Value *, ArrayRef<llvm::Value *>,
> > > ArrayRef<llvm::OperandBundleDef>, const llvm::Twine &)") at
> > > ./assert/assert.c:101
> > > #6 0x00007f5bc110f138 in llvm::CallInst::init (this=0x557a91f3e508,
> > > FTy=0x557a91ed9ae0, Func=0x557a91f8be88, Args=..., Bundles=...,
> > > NameStr=...) at
> > > /home/bf/src/llvm-project-5/llvm/lib/IR/Instructions.cpp:297
> > > #7 0x00007f5bc0fa579d in llvm::CallInst::CallInst
> > > (this=0x557a91f3e508, Ty=0x557a91ed9ae0, Func=0x557a91f8be88,
> > > Args=..., Bundles=..., NameStr=..., InsertBefore=0x0) at
> > > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1934
> > > #8 0x00007f5bc0fa538c in llvm::CallInst::Create (Ty=0x557a91ed9ae0,
> > > Func=0x557a91f8be88, Args=..., Bundles=..., NameStr=...,
> > > InsertBefore=0x0) at
> > > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/Instructions.h:1444
> > > #9 0x00007f5bc0fa51f9 in llvm::IRBuilder<llvm::ConstantFolder,
> > > llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
> > > FTy=0x557a91ed9ae0, Callee=0x557a91f8be88, Args=..., Name=...,
> > > FPMathTag=0x0) at
> > > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1669
> > > #10 0x00007f5bc100edda in llvm::IRBuilder<llvm::ConstantFolder,
> > > llvm::IRBuilderDefaultInserter>::CreateCall (this=0x557a91f9c6a0,
> > > Callee=0x557a91f8be88, Args=..., Name=..., FPMathTag=0x0) at
> > > /home/bf/src/llvm-project-5/llvm/include/llvm/IR/IRBuilder.h:1663
> > > #11 0x00007f5bc100714e in LLVMBuildCall (B=0x557a91f9c6a0,
> > > Fn=0x557a91f8be88, Args=0x7ffde6fa0b50, NumArgs=6, Name=0x7f5bc30b648c
> > > "funccall_iocoerce_in_safe") at
> > > /home/bf/src/llvm-project-5/llvm/lib/IR/Core.cpp:2964
> > > #12 0x00007f5bc30af861 in llvm_compile_expr (state=0x557a91fbeac0) at
> > > /home/bf/bf-build/pogona/HEAD/pgsql.build/../pgsql/src/backend/jit/llvm/llvmjit_expr.c:1373
> > >
> > > This seems to me to be complaining about the following addition:
> > >
> > > + {
> > > + Oid ioparam = op->d.iocoerce.typioparam;
> > > + LLVMValueRef v_params[6];
> > > + LLVMValueRef v_success;
> > > +
> > > + v_params[0] = l_ptr_const(op->d.iocoerce.finfo_in,
> > > + l_ptr(StructFmgrInfo));
> > > + v_params[1] = v_output;
> > > + v_params[2] = l_oid_const(lc, ioparam);
> > > + v_params[3] = l_int32_const(lc, -1);
> > > + v_params[4] = l_ptr_const(op->d.iocoerce.escontext,
> > > +
> > > l_ptr(StructErrorSaveContext));
> > >
> > > - LLVMBuildStore(b, v_retval, v_resvaluep);
> > > + /*
> > > + * InputFunctionCallSafe() will write directly into
> > > + * *op->resvalue.
> > > + */
> > > + v_params[5] = v_resvaluep;
> > > +
> > > + v_success = LLVMBuildCall(b, llvm_pg_func(mod,
> > > "InputFunctionCallSafe"),
> > > + v_params, lengthof(v_params),
> > > + "funccall_iocoerce_in_safe");
> > > +
> > > + /*
> > > + * Return null if InputFunctionCallSafe() encountered
> > > + * an error.
> > > + */
> > > + v_resnullp = LLVMBuildICmp(b, LLVMIntEQ, v_success,
> > > + l_sbool_const(0), "");
> > > + }
> >
> > Although most animals except pogona looked fine, I've decided to revert the patch for now.
> >
> > IIUC, LLVM is complaining that the code in the above block is not passing the arguments of InputFunctionCallSafe() using the correct types. I'm not exactly sure which particular argument is not handled correctly in the above code, but perhaps it's:
> >
> >
> > + v_params[1] = v_output;
> >
> > which maps to char *str argument of InputFunctionCallSafe(). v_output is set in the code preceding the above block as follows:
> >
> > /* and call output function (can never return NULL) */
> > v_output = LLVMBuildCall(b, v_fn_out, &v_fcinfo_out,
> > 1, "funccall_coerce_out");
> >
> > I thought that it would be fine to pass it as-is to the call of InputFunctionCallSafe() given that v_fn_out is a call to a function that returns char *, but perhaps not.
>
> OK, I think I could use some help from LLVM experts here.
>
> So, the LLVM code involving setting up a call to
> InputFunctionCallSafe() seems to *work*, but BF animal pogona's debug
> build (?) is complaining that the parameter types don't match up.
> Parameters are set up as follows:
>
> + {
> + Oid ioparam = op->d.iocoerce.typioparam;
> + LLVMValueRef v_params[6];
> + LLVMValueRef v_success;
> +
> + v_params[0] = l_ptr_const(op->d.iocoerce.finfo_in,
> + l_ptr(StructFmgrInfo));
> + v_params[1] = v_output;
> + v_params[2] = l_oid_const(lc, ioparam);
> + v_params[3] = l_int32_const(lc, -1);
> + v_params[4] = l_ptr_const(op->d.iocoerce.escontext,
> +
> l_ptr(StructErrorSaveContext));
> + /*
> + * InputFunctionCallSafe() will write directly into
> + * *op->resvalue.
> + */
> + v_params[5] = v_resvaluep;
> +
> + v_success = LLVMBuildCall(b, llvm_pg_func(mod,
> "InputFunctionCallSafe"),
> + v_params, lengthof(v_params),
> + "funccall_iocoerce_in_safe");
> +
> + /*
> + * Return null if InputFunctionCallSafe() encountered
> + * an error.
> + */
> + v_resnullp = LLVMBuildICmp(b, LLVMIntEQ, v_success,
> + l_sbool_const(0), "");
> + }
>
> And here's InputFunctionCallSafe's signature:
>
> bool
> InputFunctionCallSafe(FmgrInfo *flinfo, Datum d,
> Oid typioparam, int32 typmod,
> fmNodePtr escontext,
> Datum *result)
>
> I suspected that assignment to either param[1] or param[5] might be wrong.
>
> param[1] in InputFunctionCallSafe's signature is char *, but the code
> assigns it v_output, which is an LLVMValueRef for the output
> function's output, a Datum, so I thought LLVM's type checker is
> complaining that I'm trying to pass the Datum to char * without
> appropriate conversion.
>
> param[5] in InputFunctionCallSafe's signature is Node *, but the above
> code is assigning it an LLVMValueRef for iocoerce's escontext whose
> type is ErrorSaveContext.
>
> Maybe some other param is wrong.
>
> I tried various ways to fix both, but with no success. My way of
> checking for failure is to disassemble the IR code in .bc files
> (generated with jit_dump_bitcode) with llvm-dis and finding that it
> gives me errors such as:
>
> $ llvm-dis 58536.0.bc
> llvm-dis: error: Invalid record (Producer: 'LLVM7.0.1' Reader: 'LLVM 7.0.1')
>
> $ llvm-dis 58536.0.bc
> llvm-dis: error: Invalid cast (Producer: 'LLVM7.0.1' Reader: 'LLVM 7.0.1')
So I built LLVM sources to get asserts like pogona:
$ llvm-config --version
15.0.7
$ llvm-config --assertion-mode
ON
and I do now get a crash with bt that looks like this (not same as pogona):
#0 0x00007fe31e83c387 in raise () from /lib64/libc.so.6
#1 0x00007fe31e83da78 in abort () from /lib64/libc.so.6
#2 0x00007fe31e8351a6 in __assert_fail_base () from /lib64/libc.so.6
#3 0x00007fe31e835252 in __assert_fail () from /lib64/libc.so.6
#4 0x00007fe3136d8132 in llvm::CallInst::init(llvm::FunctionType*,
llvm::Value*, llvm::ArrayRef<llvm::Value*>,
llvm::ArrayRef<llvm::OperandBundleDefT<llvm::Value*> >, llvm::Twine
const&) ()
from /home/amit/llvm/lib/libLLVMCore.so.15
#5 0x00007fe31362137a in
llvm::IRBuilderBase::CreateCall(llvm::FunctionType*, llvm::Value*,
llvm::ArrayRef<llvm::Value*>, llvm::Twine const&, llvm::MDNode*) ()
from /home/amit/llvm/lib/libLLVMCore.so.15
#6 0x00007fe31362d627 in LLVMBuildCall () from
/home/amit/llvm/lib/libLLVMCore.so.15
#7 0x00007fe3205e7e92 in llvm_compile_expr (state=0x1114e48) at
llvmjit_expr.c:1374
#8 0x0000000000bd3fbc in jit_compile_expr (state=0x1114e48) at jit.c:177
#9 0x000000000072442b in ExecReadyExpr (state=0x1114e48) at execExpr.c:880
#10 0x000000000072387c in ExecBuildProjectionInfo
(targetList=0x1110840, econtext=0x1114a20, slot=0x1114db0,
parent=0x1114830, inputDesc=0x1114ab0) at execExpr.c:484
#11 0x000000000074e917 in ExecAssignProjectionInfo
(planstate=0x1114830, inputDesc=0x1114ab0) at execUtils.c:547
#12 0x000000000074ea02 in ExecConditionalAssignProjectionInfo
(planstate=0x1114830, inputDesc=0x1114ab0, varno=2)
at execUtils.c:585
#13 0x0000000000749814 in ExecAssignScanProjectionInfo
(node=0x1114830) at execScan.c:276
#14 0x0000000000790bf0 in ExecInitValuesScan (node=0x1045020,
estate=0x1114600, eflags=32)
at nodeValuesscan.c:257
#15 0x00000000007451c9 in ExecInitNode (node=0x1045020,
estate=0x1114600, eflags=32) at execProcnode.c:265
#16 0x000000000073a952 in InitPlan (queryDesc=0x1070760, eflags=32) at
execMain.c:968
#17 0x0000000000739828 in standard_ExecutorStart (queryDesc=0x1070760,
eflags=32) at execMain.c:266
#18 0x000000000073959d in ExecutorStart (queryDesc=0x1070760,
eflags=0) at execMain.c:145
#19 0x00000000009c1aaa in PortalStart (portal=0x10bf7d0, params=0x0,
eflags=0, snapshot=0x0) at pquery.c:517
#20 0x00000000009bbba8 in exec_simple_query (
query_string=0x10433c0 "select i::pg_lsn from (values ('x/a'),
('b/b')) a(i);") at postgres.c:1233
#21 0x00000000009c0263 in PostgresMain (dbname=0x1079750 "postgres",
username=0x1079738 "amit")
at postgres.c:4652
#22 0x00000000008f72d6 in BackendRun (port=0x106e740) at postmaster.c:4439
#23 0x00000000008f6c6f in BackendStartup (port=0x106e740) at postmaster.c:4167
#24 0x00000000008f363e in ServerLoop () at postmaster.c:1781
#25 0x00000000008f300e in PostmasterMain (argc=5, argv=0x103dc60) at
postmaster.c:1465
#26 0x00000000007bbfb4 in main (argc=5, argv=0x103dc60) at main.c:198
The LLVMBuildCall() in frame #6 is added by the patch that I also
mentioned in the previous replies. I haven't yet pinpointed down
which of the LLVM's asserts it is, nor have I been able to walk
through LLVM source code using gdb to figure what the new code is
doing wrong. Maybe I'm still missing a trick or two...
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-10-06 21:48 ` Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
1 sibling, 1 reply; 94+ messages in thread
From: Andres Freund @ 2023-10-06 21:48 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
Hi,
On 2023-09-29 13:57:46 +0900, Amit Langote wrote:
> Thanks. I will push the attached 0001 shortly.
Sorry for not looking at this earlier.
Have you done benchmarking to verify that 0001 does not cause performance
regressions? I'd not be suprised if it did. I'd split the soft-error path into
a separate opcode. For JIT it can largely be implemented using the same code,
eliding the check if it's the non-soft path. Or you can just put it into an
out-of-line function.
I don't like adding more stuff to ExprState. This one seems particularly
awkward, because it might be used by more than one level in an expression
subtree, which means you really need to save/restore old values when
recursing.
> @@ -1579,25 +1582,13 @@ ExecInitExprRec(Expr *node, ExprState *state,
>
> /* lookup the result type's input function */
> scratch.d.iocoerce.finfo_in = palloc0(sizeof(FmgrInfo));
> - scratch.d.iocoerce.fcinfo_data_in = palloc0(SizeForFunctionCallInfo(3));
> -
> getTypeInputInfo(iocoerce->resulttype,
> - &iofunc, &typioparam);
> + &iofunc, &scratch.d.iocoerce.typioparam);
> fmgr_info(iofunc, scratch.d.iocoerce.finfo_in);
> fmgr_info_set_expr((Node *) node, scratch.d.iocoerce.finfo_in);
> - InitFunctionCallInfoData(*scratch.d.iocoerce.fcinfo_data_in,
> - scratch.d.iocoerce.finfo_in,
> - 3, InvalidOid, NULL, NULL);
>
> - /*
> - * We can preload the second and third arguments for the input
> - * function, since they're constants.
> - */
> - fcinfo_in = scratch.d.iocoerce.fcinfo_data_in;
> - fcinfo_in->args[1].value = ObjectIdGetDatum(typioparam);
> - fcinfo_in->args[1].isnull = false;
> - fcinfo_in->args[2].value = Int32GetDatum(-1);
> - fcinfo_in->args[2].isnull = false;
> + /* Use the ErrorSaveContext passed by the caller. */
> + scratch.d.iocoerce.escontext = state->escontext;
>
> ExprEvalPushStep(state, &scratch);
> break;
I think it's likely that removing the optimization of not needing to set these
arguments ahead of time will result in a performance regression. Not to speak
of initializing the fcinfo from scratch on every evaluation of the expression.
I think this shouldn't not be merged as is.
> src/backend/parser/gram.y | 348 +++++-
This causes a nontrivial increase in the size of the parser (~5% in an
optimized build here), I wonder if we can do better.
> +/*
> + * Push steps to evaluate a JsonExpr and its various subsidiary expressions.
> + */
> +static void
> +ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
> + Datum *resv, bool *resnull,
> + ExprEvalStep *scratch)
> +{
> + JsonExprState *jsestate = palloc0(sizeof(JsonExprState));
> + JsonExprPreEvalState *pre_eval = &jsestate->pre_eval;
> + ListCell *argexprlc;
> + ListCell *argnamelc;
> + int skip_step_off = -1;
> + int passing_args_step_off = -1;
> + int coercion_step_off = -1;
> + int coercion_finish_step_off = -1;
> + int behavior_step_off = -1;
> + int onempty_expr_step_off = -1;
> + int onempty_jump_step_off = -1;
> + int onerror_expr_step_off = -1;
> + int onerror_jump_step_off = -1;
> + int result_coercion_jump_step_off = -1;
> + List *adjust_jumps = NIL;
> + ListCell *lc;
> + ExprEvalStep *as;
> +
> + jsestate->jsexpr = jexpr;
> +
> + /*
> + * Add steps to compute formatted_expr, pathspec, and PASSING arg
> + * expressions as things that must be evaluated *before* the actual JSON
> + * path expression.
> + */
> + ExecInitExprRec((Expr *) jexpr->formatted_expr, state,
> + &pre_eval->formatted_expr.value,
> + &pre_eval->formatted_expr.isnull);
> + ExecInitExprRec((Expr *) jexpr->path_spec, state,
> + &pre_eval->pathspec.value,
> + &pre_eval->pathspec.isnull);
> +
> + /*
> + * Before pushing steps for PASSING args, push a step to decide whether to
> + * skip evaluating the args and the JSON path expression depending on
> + * whether either of formatted_expr and pathspec is NULL; see
> + * ExecEvalJsonExprSkip().
> + */
> + scratch->opcode = EEOP_JSONEXPR_SKIP;
> + scratch->d.jsonexpr_skip.jsestate = jsestate;
> + skip_step_off = state->steps_len;
> + ExprEvalPushStep(state, scratch);
Could SKIP be implemented using EEOP_JUMP_IF_NULL with a bit of work? I see
that it sets jsestate->post_eval.jcstate, but I don't understand why it needs
to be done that way. /* ExecEvalJsonExprCoercion() depends on this. */ doesn't
explain that much.
> + /* PASSING args. */
> + jsestate->pre_eval.args = NIL;
> + passing_args_step_off = state->steps_len;
> + forboth(argexprlc, jexpr->passing_values,
> + argnamelc, jexpr->passing_names)
> + {
> + Expr *argexpr = (Expr *) lfirst(argexprlc);
> + String *argname = lfirst_node(String, argnamelc);
> + JsonPathVariable *var = palloc(sizeof(*var));
> +
> + var->name = pstrdup(argname->sval);
Why does this need to be strdup'd?
> + /* Step for the actual JSON path evaluation; see ExecEvalJsonExpr(). */
> + scratch->opcode = EEOP_JSONEXPR_PATH;
> + scratch->d.jsonexpr.jsestate = jsestate;
> + ExprEvalPushStep(state, scratch);
> +
> + /*
> + * Step to handle ON ERROR and ON EMPTY behavior. Also, to handle errors
> + * that may occur during coercion handling.
> + *
> + * See ExecEvalJsonExprBehavior().
> + */
> + scratch->opcode = EEOP_JSONEXPR_BEHAVIOR;
> + scratch->d.jsonexpr_behavior.jsestate = jsestate;
> + behavior_step_off = state->steps_len;
> + ExprEvalPushStep(state, scratch);
From what I can tell there a) can never be a step between EEOP_JSONEXPR_PATH
and EEOP_JSONEXPR_BEHAVIOR b) EEOP_JSONEXPR_PATH ends with an unconditional
branch. What's the point of the two different steps here?
>
> + EEO_CASE(EEOP_JSONEXPR_PATH)
> + {
> + /* too complex for an inline implementation */
> + ExecEvalJsonExpr(state, op, econtext);
> + EEO_NEXT();
> + }
Why does EEOP_JSONEXPR_PATH call ExecEvalJsonExpr, the names don't match...
> + EEO_CASE(EEOP_JSONEXPR_SKIP)
> + {
> + /* too complex for an inline implementation */
> + EEO_JUMP(ExecEvalJsonExprSkip(state, op));
> + }
...
> + EEO_CASE(EEOP_JSONEXPR_COERCION_FINISH)
> + {
> + /* too complex for an inline implementation */
> + EEO_JUMP(ExecEvalJsonExprCoercionFinish(state, op));
> + }
This seems to just return op->d.jsonexpr_coercion_finish.jump_coercion_error
or op->d.jsonexpr_coercion_finish.jump_coercion_done. Which makes me think
it'd be better to return a boolean? Particularly because that's how you
already implemented it for JIT (except that you did it by hardcoding the jump
step to compare to, which seems odd).
Separately, why do we even need a jump for both cases, and not just for the
error case?
> + EEO_CASE(EEOP_JSONEXPR_BEHAVIOR)
> + {
> + /* too complex for an inline implementation */
> + EEO_JUMP(ExecEvalJsonExprBehavior(state, op));
> + }
> +
> + EEO_CASE(EEOP_JSONEXPR_COERCION)
> + {
> + /* too complex for an inline implementation */
> + EEO_JUMP(ExecEvalJsonExprCoercion(state, op, econtext,
> + *op->resvalue, *op->resnull));
> + }
I wonder if this is the right design for this op - you're declaring this to be
op not worth implementing inline, yet you then have it implemented by hand for JIT.
> +/*
> + * Evaluate given JsonExpr by performing the specified JSON operation.
> + *
> + * This also populates the JsonExprPostEvalState with the information needed
> + * by the subsequent steps that handle the specified JsonBehavior.
> + */
> +void
> +ExecEvalJsonExpr(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
> +{
> + JsonExprState *jsestate = op->d.jsonexpr.jsestate;
> + JsonExprPreEvalState *pre_eval = &jsestate->pre_eval;
> + JsonExprPostEvalState *post_eval = &jsestate->post_eval;
> + JsonExpr *jexpr = jsestate->jsexpr;
> + Datum item;
> + Datum res = (Datum) 0;
> + bool resnull = true;
> + JsonPath *path;
> + bool throw_error = (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
> + bool *error = &post_eval->error;
> + bool *empty = &post_eval->empty;
> +
> + item = pre_eval->formatted_expr.value;
> + path = DatumGetJsonPathP(pre_eval->pathspec.value);
> +
> + /* Reset JsonExprPostEvalState for this evaluation. */
> + memset(post_eval, 0, sizeof(*post_eval));
> +
> + switch (jexpr->op)
> + {
> + case JSON_EXISTS_OP:
> + {
> + bool exists = JsonPathExists(item, path,
> + !throw_error ? error : NULL,
> + pre_eval->args);
> +
> + post_eval->jcstate = jsestate->result_jcstate;
> + if (*error)
> + {
> + *op->resnull = true;
> + *op->resvalue = (Datum) 0;
> + return;
> + }
> +
> + resnull = false;
> + res = BoolGetDatum(exists);
> + break;
> + }
Kinda seems there should be a EEOP_JSON_EXISTS/JSON_QUERY_OP op, instead of
implementing it all inside ExecEvalJsonExpr. I think this might obsolete
needing to rediscover that the value is null in SKIP etc?
> + case JSON_QUERY_OP:
> + res = JsonPathQuery(item, path, jexpr->wrapper, empty,
> + !throw_error ? error : NULL,
> + pre_eval->args);
> +
> + post_eval->jcstate = jsestate->result_jcstate;
> + if (*error)
> + {
> + *op->resnull = true;
> + *op->resvalue = (Datum) 0;
> + return;
> + }
> + resnull = !DatumGetPointer(res);
Shoulnd't this check empty?
FWIW, it's also pretty odd that JsonPathQuery() once
return (Datum) 0;
and later does
return PointerGetDatum(NULL);
> + case JSON_VALUE_OP:
> + {
> + JsonbValue *jbv = JsonPathValue(item, path, empty,
> + !throw_error ? error : NULL,
> + pre_eval->args);
> +
> + /* Might get overridden below by an item_jcstate. */
> + post_eval->jcstate = jsestate->result_jcstate;
> + if (*error)
> + {
> + *op->resnull = true;
> + *op->resvalue = (Datum) 0;
> + return;
> + }
> +
> + if (!jbv) /* NULL or empty */
> + {
> + resnull = true;
> + break;
> + }
> +
> + Assert(!*empty);
> +
> + resnull = false;
> +
> + /* Coerce scalar item to the output type */
> +
> + /*
> + * If the requested output type is json(b), use
> + * JsonExprState.result_coercion to do the coercion.
> + */
> + if (jexpr->returning->typid == JSONOID ||
> + jexpr->returning->typid == JSONBOID)
> + {
> + /* Use result_coercion from json[b] to the output type */
> + res = JsonbPGetDatum(JsonbValueToJsonb(jbv));
> + break;
> + }
> +
> + /*
> + * Else, use one of the item_coercions.
> + *
> + * Error out if no cast exists to coerce SQL/JSON item to the
> + * the output type.
> + */
> + res = ExecPrepareJsonItemCoercion(jbv,
> + jsestate->item_jcstates,
> + &post_eval->jcstate);
> + if (post_eval->jcstate &&
> + post_eval->jcstate->coercion &&
> + (post_eval->jcstate->coercion->via_io ||
> + post_eval->jcstate->coercion->via_populate))
> + {
> + if (!throw_error)
> + {
> + *op->resnull = true;
> + *op->resvalue = (Datum) 0;
> + return;
> + }
> +
> + /*
> + * Coercion via I/O means here that the cast to the target
> + * type simply does not exist.
> + */
> + ereport(ERROR,
> + (errcode(ERRCODE_SQL_JSON_ITEM_CANNOT_BE_CAST_TO_TARGET_TYPE),
> + errmsg("SQL/JSON item cannot be cast to target type")));
> + }
> + break;
> + }
> +
> + default:
> + elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
> + *op->resnull = true;
> + *op->resvalue = (Datum) 0;
> + return;
> + }
> +
> + /*
> + * If the ON EMPTY behavior is to cause an error, do so here. Other
> + * behaviors will be handled in ExecEvalJsonExprBehavior().
> + */
> + if (*empty)
> + {
> + Assert(jexpr->on_empty); /* it is not JSON_EXISTS */
> +
> + if (jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
> + {
> + if (!throw_error)
> + {
> + *op->resnull = true;
> + *op->resvalue = (Datum) 0;
> + return;
> + }
> +
> + ereport(ERROR,
> + (errcode(ERRCODE_NO_SQL_JSON_ITEM),
> + errmsg("no SQL/JSON item")));
> + }
> + }
> +
> + *op->resvalue = res;
> + *op->resnull = resnull;
> +}
> +
> +/*
> + * Skip calling ExecEvalJson() on the given JsonExpr?
I don't think that function exists.
> + * Returns the step address to be performed next.
> + */
> +int
> +ExecEvalJsonExprSkip(ExprState *state, ExprEvalStep *op)
> +{
> + JsonExprState *jsestate = op->d.jsonexpr_skip.jsestate;
> +
> + /*
> + * Skip if either of the input expressions has turned out to be NULL,
> + * though do execute domain checks for NULLs, which are handled by the
> + * coercion step.
> + */
> + if (jsestate->pre_eval.formatted_expr.isnull ||
> + jsestate->pre_eval.pathspec.isnull)
> + {
> + *op->resvalue = (Datum) 0;
> + *op->resnull = true;
> +
> + /* ExecEvalJsonExprCoercion() depends on this. */
> + jsestate->post_eval.jcstate = jsestate->result_jcstate;
> +
> + return op->d.jsonexpr_skip.jump_coercion;
> + }
> +
> + /*
> + * Go evaluate the PASSING args if any and subsequently JSON path itself.
> + */
> + return op->d.jsonexpr_skip.jump_passing_args;
> +}
> +
> +/*
> + * Returns the step address to perform the JsonBehavior applicable to
> + * the JSON item that resulted from evaluating the given JsonExpr.
> + *
> + * Returns the step address to be performed next.
> + */
> +int
> +ExecEvalJsonExprBehavior(ExprState *state, ExprEvalStep *op)
> +{
> + JsonExprState *jsestate = op->d.jsonexpr_behavior.jsestate;
> + JsonExprPostEvalState *post_eval = &jsestate->post_eval;
> + JsonBehavior *behavior = NULL;
> + int jump_to = -1;
> +
> + if (post_eval->error || post_eval->coercion_error)
> + {
> + behavior = jsestate->jsexpr->on_error;
> + jump_to = op->d.jsonexpr_behavior.jump_onerror_expr;
> + }
> + else if (post_eval->empty)
> + {
> + behavior = jsestate->jsexpr->on_empty;
> + jump_to = op->d.jsonexpr_behavior.jump_onempty_expr;
> + }
> + else if (!post_eval->coercion_done)
> + {
> + /*
> + * If no error or the JSON item is not empty, directly go to the
> + * coercion step to coerce the item as is.
> + */
> + return op->d.jsonexpr_behavior.jump_coercion;
> + }
> +
> + Assert(behavior);
> +
> + /*
> + * Set up for coercion step that will run to coerce a non-default behavior
> + * value. It should use result_coercion, if any. Errors that may occur
> + * should be thrown for JSON ops other than JSON_VALUE_OP.
> + */
> + if (behavior->btype != JSON_BEHAVIOR_DEFAULT)
> + {
> + post_eval->jcstate = jsestate->result_jcstate;
> + post_eval->coercing_behavior_expr = true;
> + }
> +
> + Assert(jump_to >= 0);
> + return jump_to;
> +}
> +
> +/*
> + * Evaluate or return the step address to evaluate a coercion of a JSON item
> + * to the target type. The former if the coercion must be done right away by
> + * calling the target type's input function, and for some types, by calling
> + * json_populate_type().
> + *
> + * Returns the step address to be performed next.
> + */
> +int
> +ExecEvalJsonExprCoercion(ExprState *state, ExprEvalStep *op,
> + ExprContext *econtext,
> + Datum res, bool resnull)
> +{
> + JsonExprState *jsestate = op->d.jsonexpr_coercion.jsestate;
> + JsonExpr *jexpr = jsestate->jsexpr;
> + JsonExprPostEvalState *post_eval = &jsestate->post_eval;
> + JsonCoercionState *jcstate = post_eval->jcstate;
> + char *val_string = NULL;
> + bool omit_quotes = false;
> +
> + switch (jexpr->op)
> + {
> + case JSON_EXISTS_OP:
> + if (jcstate && jcstate->jump_eval_expr >= 0)
> + return jcstate->jump_eval_expr;
Shouldn't this be a compile-time check and instead be handled by simply not
emitting a step instead?
> + /* No coercion needed. */
> + post_eval->coercion_done = true;
> + return op->d.jsonexpr_coercion.jump_coercion_done;
Which then means we also don't need to emit anything here, no?
> +/*
> + * Prepare SQL/JSON item coercion to the output type. Returned a datum of the
> + * corresponding SQL type and a pointer to the coercion state.
> + */
> +static Datum
> +ExecPrepareJsonItemCoercion(JsonbValue *item, List *item_jcstates,
> + JsonCoercionState **p_item_jcstate)
I might have missed it, but if not: The whole way the coercion stuff works
needs a decent comment explaining how things fit together.
What does "item" really mean here?
> +{
> + JsonCoercionState *item_jcstate;
> + Datum res;
> + JsonbValue buf;
> +
> + if (item->type == jbvBinary &&
> + JsonContainerIsScalar(item->val.binary.data))
> + {
> + bool res PG_USED_FOR_ASSERTS_ONLY;
> +
> + res = JsonbExtractScalar(item->val.binary.data, &buf);
> + item = &buf;
> + Assert(res);
> + }
> +
> + /* get coercion state reference and datum of the corresponding SQL type */
> + switch (item->type)
> + {
> + case jbvNull:
> + item_jcstate = list_nth(item_jcstates, JsonItemTypeNull);
This seems quite odd. We apparently have a fixed-length array, where specific
offsets have specific meanings, yet it's encoded as a list that's then
accessed with constant offsets?
Right now ExecEvalJsonExpr() stores what ExecPrepareJsonItemCoercion() chooses
in post_eval->jcstate. Which the immediately following
ExecEvalJsonExprBehavior() then digs out again. Then there's also control flow
via post_eval->coercing_behavior_expr. This is ... not nice.
ISTM that jsestate should have an array of jump targets, indexed by
item->type. Which, for llvm IR, you can encode as a switch statement, instead
of doing control flow via JsonExprState/JsonExprPostEvalState. There's
obviously a bit more needed, but I think something like that should work, and
simplify things a fair bit.
> @@ -15711,6 +15721,192 @@ func_expr_common_subexpr:
> n->location = @1;
> $$ = (Node *) n;
> }
> + | JSON_QUERY '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_wrapper_behavior
> + json_quotes_clause_opt
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_QUERY_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->wrapper = $5;
> + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> + ereport(ERROR,
> + (errcode(ERRCODE_SYNTAX_ERROR),
> + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> + parser_errposition(@6)));
> + n->quotes = $6;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
> + | JSON_QUERY '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_wrapper_behavior
> + json_quotes_clause_opt
> + json_query_behavior ON EMPTY_P
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_QUERY_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->wrapper = $5;
> + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> + ereport(ERROR,
> + (errcode(ERRCODE_SYNTAX_ERROR),
> + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> + parser_errposition(@6)));
> + n->quotes = $6;
> + n->on_empty = $7;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
> + | JSON_QUERY '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_wrapper_behavior
> + json_quotes_clause_opt
> + json_query_behavior ON ERROR_P
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_QUERY_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->wrapper = $5;
> + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> + ereport(ERROR,
> + (errcode(ERRCODE_SYNTAX_ERROR),
> + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> + parser_errposition(@6)));
> + n->quotes = $6;
> + n->on_error = $7;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
> + | JSON_QUERY '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_wrapper_behavior
> + json_quotes_clause_opt
> + json_query_behavior ON EMPTY_P
> + json_query_behavior ON ERROR_P
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_QUERY_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->wrapper = $5;
> + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> + ereport(ERROR,
> + (errcode(ERRCODE_SYNTAX_ERROR),
> + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> + parser_errposition(@6)));
> + n->quotes = $6;
> + n->on_empty = $7;
> + n->on_error = $10;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
I'm sure we can find a way to deduplicate this.
> + | JSON_EXISTS '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + ')'
> + {
> + JsonFuncExpr *p = makeNode(JsonFuncExpr);
> +
> + p->op = JSON_EXISTS_OP;
> + p->common = (JsonCommon *) $3;
> + p->output = (JsonOutput *) $4;
> + p->location = @1;
> + $$ = (Node *) p;
> + }
> + | JSON_EXISTS '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_exists_behavior ON ERROR_P
> + ')'
> + {
> + JsonFuncExpr *p = makeNode(JsonFuncExpr);
> +
> + p->op = JSON_EXISTS_OP;
> + p->common = (JsonCommon *) $3;
> + p->output = (JsonOutput *) $4;
> + p->on_error = $5;
> + p->location = @1;
> + $$ = (Node *) p;
> + }
> + | JSON_VALUE '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_VALUE_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
> +
> + | JSON_VALUE '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_value_behavior ON EMPTY_P
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_VALUE_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->on_empty = $5;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
> + | JSON_VALUE '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_value_behavior ON ERROR_P
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_VALUE_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->on_error = $5;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
> +
> + | JSON_VALUE '('
> + json_api_common_syntax
> + json_returning_clause_opt
> + json_value_behavior ON EMPTY_P
> + json_value_behavior ON ERROR_P
> + ')'
> + {
> + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> +
> + n->op = JSON_VALUE_OP;
> + n->common = (JsonCommon *) $3;
> + n->output = (JsonOutput *) $4;
> + n->on_empty = $5;
> + n->on_error = $8;
> + n->location = @1;
> + $$ = (Node *) n;
> + }
> ;
And this.
> +json_query_behavior:
> + ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
> + | NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
> + | DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
> + | EMPTY_P ARRAY { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
> + | EMPTY_P OBJECT_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
> + /* non-standard, for Oracle compatibility only */
> + | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
> + ;
> +json_exists_behavior:
> + ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
> + | TRUE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
> + | FALSE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
> + | UNKNOWN { $$ = makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
> + ;
> +
> +json_value_behavior:
> + NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
> + | ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
> + | DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
> + ;
This also seems like it could use some dedup.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
@ 2023-11-15 13:00 ` Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 04:53 ` Re: remaining sql/json patches Erik Rijkers <[email protected]>
0 siblings, 2 replies; 94+ messages in thread
From: Amit Langote @ 2023-11-15 13:00 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
Hi,
Sorry for the late reply.
On Sat, Oct 7, 2023 at 6:49 AM Andres Freund <[email protected]> wrote:
> On 2023-09-29 13:57:46 +0900, Amit Langote wrote:
> > +/*
> > + * Push steps to evaluate a JsonExpr and its various subsidiary expressions.
> > + */
> > +static void
> > +ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
> > + Datum *resv, bool *resnull,
> > + ExprEvalStep *scratch)
> > +{
> > + JsonExprState *jsestate = palloc0(sizeof(JsonExprState));
> > + JsonExprPreEvalState *pre_eval = &jsestate->pre_eval;
> > + ListCell *argexprlc;
> > + ListCell *argnamelc;
> > + int skip_step_off = -1;
> > + int passing_args_step_off = -1;
> > + int coercion_step_off = -1;
> > + int coercion_finish_step_off = -1;
> > + int behavior_step_off = -1;
> > + int onempty_expr_step_off = -1;
> > + int onempty_jump_step_off = -1;
> > + int onerror_expr_step_off = -1;
> > + int onerror_jump_step_off = -1;
> > + int result_coercion_jump_step_off = -1;
> > + List *adjust_jumps = NIL;
> > + ListCell *lc;
> > + ExprEvalStep *as;
> > +
> > + jsestate->jsexpr = jexpr;
> > +
> > + /*
> > + * Add steps to compute formatted_expr, pathspec, and PASSING arg
> > + * expressions as things that must be evaluated *before* the actual JSON
> > + * path expression.
> > + */
> > + ExecInitExprRec((Expr *) jexpr->formatted_expr, state,
> > + &pre_eval->formatted_expr.value,
> > + &pre_eval->formatted_expr.isnull);
> > + ExecInitExprRec((Expr *) jexpr->path_spec, state,
> > + &pre_eval->pathspec.value,
> > + &pre_eval->pathspec.isnull);
> > +
> > + /*
> > + * Before pushing steps for PASSING args, push a step to decide whether to
> > + * skip evaluating the args and the JSON path expression depending on
> > + * whether either of formatted_expr and pathspec is NULL; see
> > + * ExecEvalJsonExprSkip().
> > + */
> > + scratch->opcode = EEOP_JSONEXPR_SKIP;
> > + scratch->d.jsonexpr_skip.jsestate = jsestate;
> > + skip_step_off = state->steps_len;
> > + ExprEvalPushStep(state, scratch);
>
> Could SKIP be implemented using EEOP_JUMP_IF_NULL with a bit of work? I see
> that it sets jsestate->post_eval.jcstate, but I don't understand why it needs
> to be done that way. /* ExecEvalJsonExprCoercion() depends on this. */ doesn't
> explain that much.
OK, I've managed to make this work using EEOP_JUMP_IF_NULL for each of
the two expressions that need checking: formatted_expr and pathspec.
> > + /* PASSING args. */
> > + jsestate->pre_eval.args = NIL;
> > + passing_args_step_off = state->steps_len;
> > + forboth(argexprlc, jexpr->passing_values,
> > + argnamelc, jexpr->passing_names)
> > + {
> > + Expr *argexpr = (Expr *) lfirst(argexprlc);
> > + String *argname = lfirst_node(String, argnamelc);
> > + JsonPathVariable *var = palloc(sizeof(*var));
> > +
> > + var->name = pstrdup(argname->sval);
>
> Why does this need to be strdup'd?
Seems unnecessary, so removed.
> > + /* Step for the actual JSON path evaluation; see ExecEvalJsonExpr(). */
> > + scratch->opcode = EEOP_JSONEXPR_PATH;
> > + scratch->d.jsonexpr.jsestate = jsestate;
> > + ExprEvalPushStep(state, scratch);
> > +
> > + /*
> > + * Step to handle ON ERROR and ON EMPTY behavior. Also, to handle errors
> > + * that may occur during coercion handling.
> > + *
> > + * See ExecEvalJsonExprBehavior().
> > + */
> > + scratch->opcode = EEOP_JSONEXPR_BEHAVIOR;
> > + scratch->d.jsonexpr_behavior.jsestate = jsestate;
> > + behavior_step_off = state->steps_len;
> > + ExprEvalPushStep(state, scratch);
>
> From what I can tell there a) can never be a step between EEOP_JSONEXPR_PATH
> and EEOP_JSONEXPR_BEHAVIOR b) EEOP_JSONEXPR_PATH ends with an unconditional
> branch. What's the point of the two different steps here?
A separate BEHAVIOR step is needed to jump to when the coercion step
catches an error which must be handled with the appropriate ON ERROR
behavior.
> > + EEO_CASE(EEOP_JSONEXPR_PATH)
> > + {
> > + /* too complex for an inline implementation */
> > + ExecEvalJsonExpr(state, op, econtext);
> > + EEO_NEXT();
> > + }
>
> Why does EEOP_JSONEXPR_PATH call ExecEvalJsonExpr, the names don't match...
Renamed to ExecEvalJsonExprPath().
> > + EEO_CASE(EEOP_JSONEXPR_SKIP)
> > + {
> > + /* too complex for an inline implementation */
> > + EEO_JUMP(ExecEvalJsonExprSkip(state, op));
> > + }
> ...
>
>
> > + EEO_CASE(EEOP_JSONEXPR_COERCION_FINISH)
> > + {
> > + /* too complex for an inline implementation */
> > + EEO_JUMP(ExecEvalJsonExprCoercionFinish(state, op));
> > + }
>
> This seems to just return op->d.jsonexpr_coercion_finish.jump_coercion_error
> or op->d.jsonexpr_coercion_finish.jump_coercion_done. Which makes me think
> it'd be better to return a boolean? Particularly because that's how you
> already implemented it for JIT (except that you did it by hardcoding the jump
> step to compare to, which seems odd).
>
> Separately, why do we even need a jump for both cases, and not just for the
> error case?
Agreed. I've redesigned all of the steps so that we need to remember
only a couple of jump addresses in JsonExprState for hard-coded
jumping:
1. the address of the step that handles ON ERROR/EMPTY clause
(statically set during compilation)
2. the address of the step that evaluates coercion (dynamically set
depending on the type of the JSON value to coerce)
The redesign involved changing:
* What each step does
* Arranging steps in the order of operations that must be performed in
the following order:
1. compute formatted_expr
2. JUMP_IF_NULL (jumps to coerce the NULL result)
3. compute pathspec
4. JUMP_IF_NULL (jumps to coerce the NULL result)
5. compute PASSING arg expressions or noop
6. compute JsonPath{Exists|Query|Value} (hard-coded jump to step 9 if
error/empty or to appropriate coercion)
7. evaluate coercion (via expression or via IO in
ExecEvalJsonCoercionViaPopulateOrIO) ->
8. coercion finish
9. JUMP_IF_NOT_TRUE (error) (jumps to skip the next expression if !error)
10. ON ERROR expression
12. JUMP_IF_NOT_TRUE (empty) (jumps to skip the next expression if !empty)
13. ON EMPTY expression
There are also some unconditional JUMPs added in between above steps
to skip to end or the appropriate target address as needed.
> > + EEO_CASE(EEOP_JSONEXPR_BEHAVIOR)
> > + {
> > + /* too complex for an inline implementation */
> > + EEO_JUMP(ExecEvalJsonExprBehavior(state, op));
> > + }
> > +
> > + EEO_CASE(EEOP_JSONEXPR_COERCION)
> > + {
> > + /* too complex for an inline implementation */
> > + EEO_JUMP(ExecEvalJsonExprCoercion(state, op, econtext,
> > + *op->resvalue, *op->resnull));
> > + }
>
> I wonder if this is the right design for this op - you're declaring this to be
> op not worth implementing inline, yet you then have it implemented by hand for JIT.
This has been redesigned to not require the hard-coded jumps like these.
> > +/*
> > + * Evaluate given JsonExpr by performing the specified JSON operation.
> > + *
> > + * This also populates the JsonExprPostEvalState with the information needed
> > + * by the subsequent steps that handle the specified JsonBehavior.
> > + */
> > +void
> > +ExecEvalJsonExpr(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
> > +{
> > + JsonExprState *jsestate = op->d.jsonexpr.jsestate;
> > + JsonExprPreEvalState *pre_eval = &jsestate->pre_eval;
> > + JsonExprPostEvalState *post_eval = &jsestate->post_eval;
> > + JsonExpr *jexpr = jsestate->jsexpr;
> > + Datum item;
> > + Datum res = (Datum) 0;
> > + bool resnull = true;
> > + JsonPath *path;
> > + bool throw_error = (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
> > + bool *error = &post_eval->error;
> > + bool *empty = &post_eval->empty;
> > +
> > + item = pre_eval->formatted_expr.value;
> > + path = DatumGetJsonPathP(pre_eval->pathspec.value);
> > +
> > + /* Reset JsonExprPostEvalState for this evaluation. */
> > + memset(post_eval, 0, sizeof(*post_eval));
> > +
> > + switch (jexpr->op)
> > + {
> > + case JSON_EXISTS_OP:
> > + {
> > + bool exists = JsonPathExists(item, path,
> > + !throw_error ? error : NULL,
> > + pre_eval->args);
> > +
> > + post_eval->jcstate = jsestate->result_jcstate;
> > + if (*error)
> > + {
> > + *op->resnull = true;
> > + *op->resvalue = (Datum) 0;
> > + return;
> > + }
> > +
> > + resnull = false;
> > + res = BoolGetDatum(exists);
> > + break;
> > + }
>
> Kinda seems there should be a EEOP_JSON_EXISTS/JSON_QUERY_OP op, instead of
> implementing it all inside ExecEvalJsonExpr. I think this might obsolete
> needing to rediscover that the value is null in SKIP etc?
I tried but didn't really see the point of breaking
ExecEvalJsonExprPath() down into one step per JSON_*OP.
The skipping logic is based on the result of *2* input expressions
formatted_expr and pathspec which are computed before getting to
ExecEvalJsonExprPath(). Also, the skipping logic also allows to skip
the evaluation of PASSING arguments which also need to be computed
before ExecEvalJsonExprPath().
> > + case JSON_QUERY_OP:
> > + res = JsonPathQuery(item, path, jexpr->wrapper, empty,
> > + !throw_error ? error : NULL,
> > + pre_eval->args);
> > +
> > + post_eval->jcstate = jsestate->result_jcstate;
> > + if (*error)
> > + {
> > + *op->resnull = true;
> > + *op->resvalue = (Datum) 0;
> > + return;
> > + }
> > + resnull = !DatumGetPointer(res);
>
> Shoulnd't this check empty?
Fixed.
> FWIW, it's also pretty odd that JsonPathQuery() once
> return (Datum) 0;
> and later does
> return PointerGetDatum(NULL);
Yes, fixed to use the former style at all returns.
> > + case JSON_VALUE_OP:
> > + {
> > + JsonbValue *jbv = JsonPathValue(item, path, empty,
> > + !throw_error ? error : NULL,
> > + pre_eval->args);
> > +
> > + /* Might get overridden below by an item_jcstate. */
> > + post_eval->jcstate = jsestate->result_jcstate;
> > + if (*error)
> > + {
> > + *op->resnull = true;
> > + *op->resvalue = (Datum) 0;
> > + return;
> > + }
> > +
> > + if (!jbv) /* NULL or empty */
> > + {
> > + resnull = true;
> > + break;
> > + }
> > +
> > + Assert(!*empty);
> > +
> > + resnull = false;
> > +
> > + /* Coerce scalar item to the output type */
> > +
> > + /*
> > + * If the requested output type is json(b), use
> > + * JsonExprState.result_coercion to do the coercion.
> > + */
> > + if (jexpr->returning->typid == JSONOID ||
> > + jexpr->returning->typid == JSONBOID)
> > + {
> > + /* Use result_coercion from json[b] to the output type */
> > + res = JsonbPGetDatum(JsonbValueToJsonb(jbv));
> > + break;
> > + }
> > +
> > + /*
> > + * Else, use one of the item_coercions.
> > + *
> > + * Error out if no cast exists to coerce SQL/JSON item to the
> > + * the output type.
> > + */
> > + res = ExecPrepareJsonItemCoercion(jbv,
> > + jsestate->item_jcstates,
> > + &post_eval->jcstate);
> > + if (post_eval->jcstate &&
> > + post_eval->jcstate->coercion &&
> > + (post_eval->jcstate->coercion->via_io ||
> > + post_eval->jcstate->coercion->via_populate))
> > + {
> > + if (!throw_error)
> > + {
> > + *op->resnull = true;
> > + *op->resvalue = (Datum) 0;
> > + return;
> > + }
> > +
> > + /*
> > + * Coercion via I/O means here that the cast to the target
> > + * type simply does not exist.
> > + */
> > + ereport(ERROR,
> > + (errcode(ERRCODE_SQL_JSON_ITEM_CANNOT_BE_CAST_TO_TARGET_TYPE),
> > + errmsg("SQL/JSON item cannot be cast to target type")));
> > + }
> > + break;
> > + }
> > +
> > + default:
> > + elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
> > + *op->resnull = true;
> > + *op->resvalue = (Datum) 0;
> > + return;
> > + }
> > +
> > + /*
> > + * If the ON EMPTY behavior is to cause an error, do so here. Other
> > + * behaviors will be handled in ExecEvalJsonExprBehavior().
> > + */
> > + if (*empty)
> > + {
> > + Assert(jexpr->on_empty); /* it is not JSON_EXISTS */
> > +
> > + if (jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
> > + {
> > + if (!throw_error)
> > + {
> > + *op->resnull = true;
> > + *op->resvalue = (Datum) 0;
> > + return;
> > + }
> > +
> > + ereport(ERROR,
> > + (errcode(ERRCODE_NO_SQL_JSON_ITEM),
> > + errmsg("no SQL/JSON item")));
> > + }
> > + }
> > +
> > + *op->resvalue = res;
> > + *op->resnull = resnull;
> > +}
> > +
> > +/*
> > + * Skip calling ExecEvalJson() on the given JsonExpr?
>
> I don't think that function exists.
Fixed.
> > + * Returns the step address to be performed next.
> > + */
> > +int
> > +ExecEvalJsonExprSkip(ExprState *state, ExprEvalStep *op)
> > +{
> > + JsonExprState *jsestate = op->d.jsonexpr_skip.jsestate;
> > +
> > + /*
> > + * Skip if either of the input expressions has turned out to be NULL,
> > + * though do execute domain checks for NULLs, which are handled by the
> > + * coercion step.
> > + */
> > + if (jsestate->pre_eval.formatted_expr.isnull ||
> > + jsestate->pre_eval.pathspec.isnull)
> > + {
> > + *op->resvalue = (Datum) 0;
> > + *op->resnull = true;
> > +
> > + /* ExecEvalJsonExprCoercion() depends on this. */
> > + jsestate->post_eval.jcstate = jsestate->result_jcstate;
> > +
> > + return op->d.jsonexpr_skip.jump_coercion;
> > + }
> > +
> > + /*
> > + * Go evaluate the PASSING args if any and subsequently JSON path itself.
> > + */
> > + return op->d.jsonexpr_skip.jump_passing_args;
> > +}
> > +
> > +/*
> > + * Returns the step address to perform the JsonBehavior applicable to
> > + * the JSON item that resulted from evaluating the given JsonExpr.
> > + *
> > + * Returns the step address to be performed next.
> > + */
> > +int
> > +ExecEvalJsonExprBehavior(ExprState *state, ExprEvalStep *op)
> > +{
> > + JsonExprState *jsestate = op->d.jsonexpr_behavior.jsestate;
> > + JsonExprPostEvalState *post_eval = &jsestate->post_eval;
> > + JsonBehavior *behavior = NULL;
> > + int jump_to = -1;
> > +
> > + if (post_eval->error || post_eval->coercion_error)
> > + {
> > + behavior = jsestate->jsexpr->on_error;
> > + jump_to = op->d.jsonexpr_behavior.jump_onerror_expr;
> > + }
> > + else if (post_eval->empty)
> > + {
> > + behavior = jsestate->jsexpr->on_empty;
> > + jump_to = op->d.jsonexpr_behavior.jump_onempty_expr;
> > + }
> > + else if (!post_eval->coercion_done)
> > + {
> > + /*
> > + * If no error or the JSON item is not empty, directly go to the
> > + * coercion step to coerce the item as is.
> > + */
> > + return op->d.jsonexpr_behavior.jump_coercion;
> > + }
> > +
> > + Assert(behavior);
> > +
> > + /*
> > + * Set up for coercion step that will run to coerce a non-default behavior
> > + * value. It should use result_coercion, if any. Errors that may occur
> > + * should be thrown for JSON ops other than JSON_VALUE_OP.
> > + */
> > + if (behavior->btype != JSON_BEHAVIOR_DEFAULT)
> > + {
> > + post_eval->jcstate = jsestate->result_jcstate;
> > + post_eval->coercing_behavior_expr = true;
> > + }
> > +
> > + Assert(jump_to >= 0);
> > + return jump_to;
> > +}
> > +
> > +/*
> > + * Evaluate or return the step address to evaluate a coercion of a JSON item
> > + * to the target type. The former if the coercion must be done right away by
> > + * calling the target type's input function, and for some types, by calling
> > + * json_populate_type().
> > + *
> > + * Returns the step address to be performed next.
> > + */
> > +int
> > +ExecEvalJsonExprCoercion(ExprState *state, ExprEvalStep *op,
> > + ExprContext *econtext,
> > + Datum res, bool resnull)
> > +{
> > + JsonExprState *jsestate = op->d.jsonexpr_coercion.jsestate;
> > + JsonExpr *jexpr = jsestate->jsexpr;
> > + JsonExprPostEvalState *post_eval = &jsestate->post_eval;
> > + JsonCoercionState *jcstate = post_eval->jcstate;
> > + char *val_string = NULL;
> > + bool omit_quotes = false;
> > +
> > + switch (jexpr->op)
> > + {
> > + case JSON_EXISTS_OP:
> > + if (jcstate && jcstate->jump_eval_expr >= 0)
> > + return jcstate->jump_eval_expr;
>
> Shouldn't this be a compile-time check and instead be handled by simply not
> emitting a step instead?
Yes and...
> > + /* No coercion needed. */
> > + post_eval->coercion_done = true;
> > + return op->d.jsonexpr_coercion.jump_coercion_done;
>
> Which then means we also don't need to emit anything here, no?
Yes.
Basically all jump address selection logic is now handled in
ExecInitJsonExpr() (compile-time) as described above.
> > +/*
> > + * Prepare SQL/JSON item coercion to the output type. Returned a datum of the
> > + * corresponding SQL type and a pointer to the coercion state.
> > + */
> > +static Datum
> > +ExecPrepareJsonItemCoercion(JsonbValue *item, List *item_jcstates,
> > + JsonCoercionState **p_item_jcstate)
>
> I might have missed it, but if not: The whole way the coercion stuff works
> needs a decent comment explaining how things fit together.
>
> What does "item" really mean here?
This term "item" I think refers to the JsonbValue returned by
JsonPathValue(), which can be one of jbvType types. Because we need
multiple coercions to account for that, I assume the original authors
decided to use the term/phrase "item coercions" to distinguish from
the result_coercion which assumes either a Boolean (EXISTS) or
Jsonb/text (QUERY) result.
> > +{
> > + JsonCoercionState *item_jcstate;
> > + Datum res;
> > + JsonbValue buf;
> > +
> > + if (item->type == jbvBinary &&
> > + JsonContainerIsScalar(item->val.binary.data))
> > + {
> > + bool res PG_USED_FOR_ASSERTS_ONLY;
> > +
> > + res = JsonbExtractScalar(item->val.binary.data, &buf);
> > + item = &buf;
> > + Assert(res);
> > + }
> > +
> > + /* get coercion state reference and datum of the corresponding SQL type */
> > + switch (item->type)
> > + {
> > + case jbvNull:
> > + item_jcstate = list_nth(item_jcstates, JsonItemTypeNull);
>
> This seems quite odd. We apparently have a fixed-length array, where specific
> offsets have specific meanings, yet it's encoded as a list that's then
> accessed with constant offsets?
>
> Right now ExecEvalJsonExpr() stores what ExecPrepareJsonItemCoercion() chooses
> in post_eval->jcstate. Which the immediately following
> ExecEvalJsonExprBehavior() then digs out again. Then there's also control flow
> via post_eval->coercing_behavior_expr. This is ... not nice.
Agree.
In the new code, the struct JsonCoercionState is gone. So any given
coercion boils down to a steps address during runtime, which is
determined by ExecEvalJsonExprPath().
> ISTM that jsestate should have an array of jump targets, indexed by
> item->type.
Yes, an array of jump targets seems better for these "item coercions".
result_coercion is a single address stored separately.
> Which, for llvm IR, you can encode as a switch statement, instead
> of doing control flow via JsonExprState/JsonExprPostEvalState. There's
> obviously a bit more needed, but I think something like that should work, and
> simplify things a fair bit.
Thanks for suggesting the switch-case idea. The LLVM IR for
EEOP_JSONEXPR_PATH now includes one to jump to one of the coercion
addresses between that for result_coercion and "item coercions" if
present.
> > @@ -15711,6 +15721,192 @@ func_expr_common_subexpr:
> > n->location = @1;
> > $$ = (Node *) n;
> > }
> > + | JSON_QUERY '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_wrapper_behavior
> > + json_quotes_clause_opt
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_QUERY_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->wrapper = $5;
> > + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> > + ereport(ERROR,
> > + (errcode(ERRCODE_SYNTAX_ERROR),
> > + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> > + parser_errposition(@6)));
> > + n->quotes = $6;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
> > + | JSON_QUERY '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_wrapper_behavior
> > + json_quotes_clause_opt
> > + json_query_behavior ON EMPTY_P
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_QUERY_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->wrapper = $5;
> > + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> > + ereport(ERROR,
> > + (errcode(ERRCODE_SYNTAX_ERROR),
> > + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> > + parser_errposition(@6)));
> > + n->quotes = $6;
> > + n->on_empty = $7;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
> > + | JSON_QUERY '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_wrapper_behavior
> > + json_quotes_clause_opt
> > + json_query_behavior ON ERROR_P
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_QUERY_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->wrapper = $5;
> > + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> > + ereport(ERROR,
> > + (errcode(ERRCODE_SYNTAX_ERROR),
> > + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> > + parser_errposition(@6)));
> > + n->quotes = $6;
> > + n->on_error = $7;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
> > + | JSON_QUERY '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_wrapper_behavior
> > + json_quotes_clause_opt
> > + json_query_behavior ON EMPTY_P
> > + json_query_behavior ON ERROR_P
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_QUERY_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->wrapper = $5;
> > + if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
> > + ereport(ERROR,
> > + (errcode(ERRCODE_SYNTAX_ERROR),
> > + errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
> > + parser_errposition(@6)));
> > + n->quotes = $6;
> > + n->on_empty = $7;
> > + n->on_error = $10;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
>
> I'm sure we can find a way to deduplicate this.
>
>
> > + | JSON_EXISTS '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + ')'
> > + {
> > + JsonFuncExpr *p = makeNode(JsonFuncExpr);
> > +
> > + p->op = JSON_EXISTS_OP;
> > + p->common = (JsonCommon *) $3;
> > + p->output = (JsonOutput *) $4;
> > + p->location = @1;
> > + $$ = (Node *) p;
> > + }
> > + | JSON_EXISTS '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_exists_behavior ON ERROR_P
> > + ')'
> > + {
> > + JsonFuncExpr *p = makeNode(JsonFuncExpr);
> > +
> > + p->op = JSON_EXISTS_OP;
> > + p->common = (JsonCommon *) $3;
> > + p->output = (JsonOutput *) $4;
> > + p->on_error = $5;
> > + p->location = @1;
> > + $$ = (Node *) p;
> > + }
> > + | JSON_VALUE '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_VALUE_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
> > +
> > + | JSON_VALUE '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_value_behavior ON EMPTY_P
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_VALUE_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->on_empty = $5;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
> > + | JSON_VALUE '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_value_behavior ON ERROR_P
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_VALUE_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->on_error = $5;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
> > +
> > + | JSON_VALUE '('
> > + json_api_common_syntax
> > + json_returning_clause_opt
> > + json_value_behavior ON EMPTY_P
> > + json_value_behavior ON ERROR_P
> > + ')'
> > + {
> > + JsonFuncExpr *n = makeNode(JsonFuncExpr);
> > +
> > + n->op = JSON_VALUE_OP;
> > + n->common = (JsonCommon *) $3;
> > + n->output = (JsonOutput *) $4;
> > + n->on_empty = $5;
> > + n->on_error = $8;
> > + n->location = @1;
> > + $$ = (Node *) n;
> > + }
> > ;
>
> And this.
>
>
>
> > +json_query_behavior:
> > + ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
> > + | NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
> > + | DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
> > + | EMPTY_P ARRAY { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
> > + | EMPTY_P OBJECT_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
> > + /* non-standard, for Oracle compatibility only */
> > + | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
> > + ;
>
>
> > +json_exists_behavior:
> > + ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
> > + | TRUE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
> > + | FALSE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
> > + | UNKNOWN { $$ = makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
> > + ;
> > +
> > +json_value_behavior:
> > + NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
> > + | ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
> > + | DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
> > + ;
>
> This also seems like it could use some dedup.
>
> > src/backend/parser/gram.y | 348 +++++-
I've given that a try and managed to reduce the gram.y footprint down to:
src/backend/parser/gram.y | 217 +++-
> This causes a nontrivial increase in the size of the parser (~5% in an
> optimized build here), I wonder if we can do better.
Hmm, sorry if I sound ignorant but what do you mean by the parser here?
I can see that the byte-size of gram.o increases by 1.66% after the
above additions (1.72% with previous versions). I've also checked
using log_parser_stats that there isn't much slowdown in the
raw-parsing speed.
Attached updated patch. The version of 0001 that I posted on Oct 11
to add the error-safe version of CoerceViaIO contained many
unnecessary bits that are now removed.
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v24-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch (2.3K, ../../CA+HiwqG_XFksg7HN_gbmcOQU00Q5bvTKeLSZ3NftewdBRw4fKA@mail.gmail.com/2-v24-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch)
download | inline diff:
From b4a9435e52ddfe93bc099733f7d1490634211de5 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:18 +0900
Subject: [PATCH v24 5/5] Claim SQL standard compliance for SQL/JSON features
Discussion: https://postgr.es/m/[email protected]
---
src/backend/catalog/sql_features.txt | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt
index 80c40eaf57..9500a80f4d 100644
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -548,15 +548,15 @@ T811 Basic SQL/JSON constructor functions YES
T812 SQL/JSON: JSON_OBJECTAGG YES
T813 SQL/JSON: JSON_ARRAYAGG with ORDER BY YES
T814 Colon in JSON_OBJECT or JSON_OBJECTAGG YES
-T821 Basic SQL/JSON query operators NO
+T821 Basic SQL/JSON query operators YES
T822 SQL/JSON: IS JSON WITH UNIQUE KEYS predicate YES
-T823 SQL/JSON: PASSING clause NO
-T824 JSON_TABLE: specific PLAN clause NO
-T825 SQL/JSON: ON EMPTY and ON ERROR clauses NO
-T826 General value expression in ON ERROR or ON EMPTY clauses NO
-T827 JSON_TABLE: sibling NESTED COLUMNS clauses NO
-T828 JSON_QUERY NO
-T829 JSON_QUERY: array wrapper options NO
+T823 SQL/JSON: PASSING clause YES
+T824 JSON_TABLE: specific PLAN clause YES
+T825 SQL/JSON: ON EMPTY and ON ERROR clauses YES
+T826 General value expression in ON ERROR or ON EMPTY clauses YES
+T827 JSON_TABLE: sibling NESTED COLUMNS clauses YES
+T828 JSON_QUERY YES
+T829 JSON_QUERY: array wrapper options YES
T830 Enforcing unique keys in SQL/JSON constructor functions YES
T831 SQL/JSON path language: strict mode YES
T832 SQL/JSON path language: item method YES
@@ -565,7 +565,7 @@ T834 SQL/JSON path language: wildcard member accessor YES
T835 SQL/JSON path language: filter expressions YES
T836 SQL/JSON path language: starts with predicate YES
T837 SQL/JSON path language: regex_like predicate YES
-T838 JSON_TABLE: PLAN DEFAULT clause NO
+T838 JSON_TABLE: PLAN DEFAULT clause YES
T839 Formatted cast of datetimes to/from character strings NO
T840 Hex integer literals in SQL/JSON path language YES
T851 SQL/JSON: optional keywords for default syntax YES
--
2.35.3
[application/octet-stream] v24-0001-Add-soft-error-handling-to-some-expression-nodes.patch (9.1K, ../../CA+HiwqG_XFksg7HN_gbmcOQU00Q5bvTKeLSZ3NftewdBRw4fKA@mail.gmail.com/3-v24-0001-Add-soft-error-handling-to-some-expression-nodes.patch)
download | inline diff:
From 05ca3f9a8776676d7d113b1f0dcf1f1fb815e41e Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Fri, 29 Sep 2023 13:22:15 +0900
Subject: [PATCH v24 1/5] Add soft error handling to some expression nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This adjusts the CoerceViaIO and CoerceToDomain expression evaluation
code to handle errors softly.
For CoerceViaIo, this adds a new ExprEvalStep opcode
EEOP_IOCOERCE_SAFE, which is implemented in new function
ExecEvalCoerceViaIOSafe(). The only difference from EEOP_IOCOERCE's
inline implementation is that the input function receives an
ErrorSaveContext via the function's FunctionCallInfo.context, which
it can use to handle errors softly.
For CoerceToDomain, this simply entails replacing the ereport() in
ExecEvalConstraintCheck() by errsave() using the ErrorSaveContext
passed in the expression's ExprEvalStep.
In both cases, the ErrorSaveContext to be used is passed by setting
ExprState.escontext to point to it before calling ExecInitExprRec()
on the expression tree whose errors are to be suppressed.
Note that no call site of ExecInitExprRec() has been changed in this
commit, so there's no functional change. This is intended for
implementing new SQL/JSON expression nodes in future commits that
will use to it suppress errors that may occur during type coercions.
Reviewed-by: Álvaro Herrera
Reviewed-by: Andres Freund
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/executor/execExpr.c | 8 ++-
src/backend/executor/execExprInterp.c | 72 ++++++++++++++++++++++++++-
src/backend/jit/llvm/llvmjit_expr.c | 6 +++
src/backend/jit/llvm/llvmjit_types.c | 1 +
src/include/executor/execExpr.h | 4 ++
src/include/nodes/execnodes.h | 7 +++
6 files changed, 96 insertions(+), 2 deletions(-)
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 2c62b0c9c8..34bd2102b5 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -1563,7 +1563,10 @@ ExecInitExprRec(Expr *node, ExprState *state,
* We don't check permissions here as a type's input/output
* function are assumed to be executable by everyone.
*/
- scratch.opcode = EEOP_IOCOERCE;
+ if (state->escontext == NULL)
+ scratch.opcode = EEOP_IOCOERCE;
+ else
+ scratch.opcode = EEOP_IOCOERCE_SAFE;
/* lookup the source type's output function */
scratch.d.iocoerce.finfo_out = palloc0(sizeof(FmgrInfo));
@@ -1599,6 +1602,8 @@ ExecInitExprRec(Expr *node, ExprState *state,
fcinfo_in->args[2].value = Int32GetDatum(-1);
fcinfo_in->args[2].isnull = false;
+ fcinfo_in->context = (Node *) state->escontext;
+
ExprEvalPushStep(state, &scratch);
break;
}
@@ -3306,6 +3311,7 @@ ExecInitCoerceToDomain(ExprEvalStep *scratch, CoerceToDomain *ctest,
/* we'll allocate workspace only if needed */
scratch->d.domaincheck.checkvalue = NULL;
scratch->d.domaincheck.checknull = NULL;
+ scratch->d.domaincheck.escontext = state->escontext;
/*
* Evaluate argument - it's fine to directly store it into resv/resnull,
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 24c2b60c62..4e152fdfe3 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -63,6 +63,7 @@
#include "executor/nodeSubplan.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/miscnodes.h"
#include "nodes/nodeFuncs.h"
#include "parser/parsetree.h"
#include "pgstat.h"
@@ -452,6 +453,7 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_CASE_TESTVAL,
&&CASE_EEOP_MAKE_READONLY,
&&CASE_EEOP_IOCOERCE,
+ &&CASE_EEOP_IOCOERCE_SAFE,
&&CASE_EEOP_DISTINCT,
&&CASE_EEOP_NOT_DISTINCT,
&&CASE_EEOP_NULLIF,
@@ -1205,6 +1207,12 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_IOCOERCE_SAFE)
+ {
+ ExecEvalCoerceViaIOSafe(state, op);
+ EEO_NEXT();
+ }
+
EEO_CASE(EEOP_DISTINCT)
{
/*
@@ -2510,6 +2518,68 @@ ExecEvalParamExtern(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
errmsg("no value found for parameter %d", paramId)));
}
+/*
+ * Evaluate a CoerceViaIO node in soft-error mode.
+ *
+ * The source value is in op's result variable.
+ */
+void
+ExecEvalCoerceViaIOSafe(ExprState *state, ExprEvalStep *op)
+{
+ char *str;
+
+ /* call output function (similar to OutputFunctionCall) */
+ if (*op->resnull)
+ {
+ /* output functions are not called on nulls */
+ str = NULL;
+ }
+ else
+ {
+ FunctionCallInfo fcinfo_out;
+
+ fcinfo_out = op->d.iocoerce.fcinfo_data_out;
+ fcinfo_out->args[0].value = *op->resvalue;
+ fcinfo_out->args[0].isnull = false;
+
+ fcinfo_out->isnull = false;
+ str = DatumGetCString(FunctionCallInvoke(fcinfo_out));
+
+ /* OutputFunctionCall assumes result isn't null */
+ Assert(!fcinfo_out->isnull);
+ }
+
+ /* call input function (similar to InputFunctionCall) */
+ if (!op->d.iocoerce.finfo_in->fn_strict || str != NULL)
+ {
+ FunctionCallInfo fcinfo_in;
+
+ fcinfo_in = op->d.iocoerce.fcinfo_data_in;
+ fcinfo_in->args[0].value = PointerGetDatum(str);
+ fcinfo_in->args[0].isnull = *op->resnull;
+ /* second and third arguments are already set up */
+
+ /* ErrorSaveContext must be present. */
+ Assert(IsA(fcinfo_in->context, ErrorSaveContext));
+
+ fcinfo_in->isnull = false;
+ *op->resvalue = FunctionCallInvoke(fcinfo_in);
+
+ if (SOFT_ERROR_OCCURRED(fcinfo_in->context))
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ /* Should get null result if and only if str is NULL */
+ if (str == NULL)
+ Assert(*op->resnull);
+ else
+ Assert(!*op->resnull);
+ }
+}
+
/*
* Evaluate a SQLValueFunction expression.
*/
@@ -3745,7 +3815,7 @@ ExecEvalConstraintCheck(ExprState *state, ExprEvalStep *op)
{
if (!*op->d.domaincheck.checknull &&
!DatumGetBool(*op->d.domaincheck.checkvalue))
- ereport(ERROR,
+ errsave((Node *) op->d.domaincheck.escontext,
(errcode(ERRCODE_CHECK_VIOLATION),
errmsg("value for domain %s violates check constraint \"%s\"",
format_type_be(op->d.domaincheck.resulttype),
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index a3a0876bff..81856a9dc7 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1431,6 +1431,12 @@ llvm_compile_expr(ExprState *state)
break;
}
+ case EEOP_IOCOERCE_SAFE:
+ build_EvalXFunc(b, mod, "ExecEvalCoerceViaIOSafe",
+ v_state, op);
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
case EEOP_DISTINCT:
case EEOP_NOT_DISTINCT:
{
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 791902ff1f..3a4be09e50 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -162,6 +162,7 @@ void *referenced_functions[] =
ExecEvalRow,
ExecEvalRowNotNull,
ExecEvalRowNull,
+ ExecEvalCoerceViaIOSafe,
ExecEvalSQLValueFunction,
ExecEvalScalarArrayOp,
ExecEvalHashedScalarArrayOp,
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index 048573c2bc..c4fd933154 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -16,6 +16,7 @@
#include "executor/nodeAgg.h"
#include "nodes/execnodes.h"
+#include "nodes/miscnodes.h"
/* forward references to avoid circularity */
struct ExprEvalStep;
@@ -168,6 +169,7 @@ typedef enum ExprEvalOp
/* evaluate assorted special-purpose expression types */
EEOP_IOCOERCE,
+ EEOP_IOCOERCE_SAFE,
EEOP_DISTINCT,
EEOP_NOT_DISTINCT,
EEOP_NULLIF,
@@ -547,6 +549,7 @@ typedef struct ExprEvalStep
bool *checknull;
/* OID of domain type */
Oid resulttype;
+ ErrorSaveContext *escontext;
} domaincheck;
/* for EEOP_CONVERT_ROWTYPE */
@@ -776,6 +779,7 @@ extern void ExecEvalParamExec(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalParamExtern(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
+extern void ExecEvalCoerceViaIOSafe(ExprState *state, ExprEvalStep *op);
extern void ExecEvalSQLValueFunction(ExprState *state, ExprEvalStep *op);
extern void ExecEvalCurrentOfExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalNextValueExpr(ExprState *state, ExprEvalStep *op);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 5d7f17dee0..6a7118d300 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -34,6 +34,7 @@
#include "fmgr.h"
#include "lib/ilist.h"
#include "lib/pairingheap.h"
+#include "nodes/miscnodes.h"
#include "nodes/params.h"
#include "nodes/plannodes.h"
#include "nodes/tidbitmap.h"
@@ -129,6 +130,12 @@ typedef struct ExprState
Datum *innermost_domainval;
bool *innermost_domainnull;
+
+ /*
+ * For expression nodes that support soft errors. Should be set to NULL
+ * before calling ExecInitExprRec() if the caller wants errors thrown.
+ */
+ ErrorSaveContext *escontext;
} ExprState;
--
2.35.3
[application/octet-stream] v24-0002-Add-soft-error-handling-to-populate_record_field.patch (23.2K, ../../CA+HiwqG_XFksg7HN_gbmcOQU00Q5bvTKeLSZ3NftewdBRw4fKA@mail.gmail.com/4-v24-0002-Add-soft-error-handling-to-populate_record_field.patch)
download | inline diff:
From 88d92a7c03d7f6a345f8c86c8a852a76ab858623 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Fri, 29 Sep 2023 13:41:28 +0900
Subject: [PATCH v24 2/5] Add soft error handling to populate_record_field()
An uncoming patch would like the ability to call it from the
executor for some SQL/JSON expression nodes and ask to suppress any
errors that may occur.
This commit does two things mainly:
* It modifies the various interfaces internal to jsonfuncs.c to pass
the ErrorSaveContext around.
* Make necessary modifications to handle the cases where the
processing is aborted partway through various functions that take
an ErrorSaveContext when a soft error occurs.
Note that the above changes are only intended to suppress errors in
the functions in jsonfuncs.c, but not those in any external functions
that the functions in jsonfuncs.c in turn call, such as those from
arrayfuncs.c. It is assumed that the various populate_* functions
validate the data before passing those to external functions.
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/utils/adt/jsonfuncs.c | 280 ++++++++++++++++++++++--------
1 file changed, 210 insertions(+), 70 deletions(-)
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index aa37c401e5..1574ed5985 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -265,6 +265,7 @@ typedef struct PopulateArrayContext
int *dims; /* dimensions */
int *sizes; /* current dimension counters */
int ndims; /* number of dimensions */
+ Node *escontext; /* For soft-error handling */
} PopulateArrayContext;
/* state for populate_array_json() */
@@ -389,7 +390,8 @@ static JsonParseErrorType elements_array_element_end(void *state, bool isnull);
static JsonParseErrorType elements_scalar(void *state, char *token, JsonTokenType tokentype);
/* turn a json object into a hash table */
-static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname);
+static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext);
/* semantic actions for populate_array_json */
static JsonParseErrorType populate_array_object_start(void *_state);
@@ -431,37 +433,42 @@ static Datum populate_record_worker(FunctionCallInfo fcinfo, const char *funcnam
/* helper functions for populate_record[set] */
static HeapTupleHeader populate_record(TupleDesc tupdesc, RecordIOData **record_p,
HeapTupleHeader defaultval, MemoryContext mcxt,
- JsObject *obj);
+ JsObject *obj, Node *escontext);
static void get_record_type_from_argument(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
static void get_record_type_from_query(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
-static void JsValueToJsObject(JsValue *jsv, JsObject *jso);
+static bool JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext);
static Datum populate_composite(CompositeIOData *io, Oid typid,
const char *colname, MemoryContext mcxt,
- HeapTupleHeader defaultval, JsValue *jsv, bool isnull);
-static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv);
+ HeapTupleHeader defaultval, JsValue *jsv, bool *isnull,
+ Node *escontext);
+static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext);
static void prepare_column_cache(ColumnIOData *column, Oid typid, int32 typmod,
MemoryContext mcxt, bool need_scalar);
static Datum populate_record_field(ColumnIOData *col, Oid typid, int32 typmod,
const char *colname, MemoryContext mcxt, Datum defaultval,
- JsValue *jsv, bool *isnull);
+ JsValue *jsv, bool *isnull, Node *escontext);
static RecordIOData *allocate_record_info(MemoryContext mcxt, int ncolumns);
static bool JsObjectGetField(JsObject *obj, char *field, JsValue *jsv);
static void populate_recordset_record(PopulateRecordsetState *state, JsObject *obj);
-static void populate_array_json(PopulateArrayContext *ctx, char *json, int len);
-static void populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
+static bool populate_array_json(PopulateArrayContext *ctx, char *json, int len);
+static bool populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
int ndim);
static void populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim);
-static void populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
-static void populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
-static void populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
+static bool populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
+static bool populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
+static bool populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
static Datum populate_array(ArrayIOData *aio, const char *colname,
- MemoryContext mcxt, JsValue *jsv);
+ MemoryContext mcxt, JsValue *jsv,
+ bool *isnull,
+ Node *escontext);
static Datum populate_domain(DomainIOData *io, Oid typid, const char *colname,
- MemoryContext mcxt, JsValue *jsv, bool isnull);
+ MemoryContext mcxt, JsValue *jsv, bool isnull,
+ Node *escontext);
/* functions supporting jsonb_delete, jsonb_set and jsonb_concat */
static JsonbValue *IteratorConcat(JsonbIterator **it1, JsonbIterator **it2,
@@ -2484,12 +2491,12 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
if (ndim <= 0)
{
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the value of key \"%s\".", ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array")));
}
@@ -2506,13 +2513,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
appendStringInfo(&indices, "[%d]", ctx->sizes[i]);
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s of key \"%s\".",
indices.data, ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s.",
@@ -2520,8 +2527,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
}
}
-/* set the number of dimensions of the populated array when it becomes known */
-static void
+/*
+ * Validate and set ndims for populating an array with some
+ * populate_array_*() function.
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
{
int i;
@@ -2529,7 +2541,12 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
Assert(ctx->ndims <= 0);
if (ndims <= 0)
+ {
populate_array_report_expected_array(ctx, ndims);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
ctx->ndims = ndims;
ctx->dims = palloc(sizeof(int) * ndims);
@@ -2537,10 +2554,16 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
for (i = 0; i < ndims; i++)
ctx->dims[i] = -1; /* dimensions are unknown yet */
+
+ return true;
}
-/* check the populated subarray dimension */
-static void
+/*
+ * Check the populated subarray dimension
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
{
int dim = ctx->sizes[ndim]; /* current dimension counter */
@@ -2548,21 +2571,31 @@ populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
if (ctx->dims[ndim] == -1)
ctx->dims[ndim] = dim; /* assign dimension if not yet known */
else if (ctx->dims[ndim] != dim)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed JSON array"),
errdetail("Multidimensional arrays must have "
"sub-arrays with matching dimensions.")));
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
+
/* reset the current array dimension size counter */
ctx->sizes[ndim] = 0;
/* increment the parent dimension counter if it is a nested sub-array */
if (ndim > 0)
ctx->sizes[ndim - 1]++;
+
+ return true;
}
-static void
+/*
+ * Returns true if the array element value was successfully extracted from jsv
+ * and added to ctx->astate. False if an error occurred when doing so.
+ */
+static bool
populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
{
Datum element;
@@ -2573,13 +2606,18 @@ populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
ctx->aio->element_type,
ctx->aio->element_typmod,
NULL, ctx->mcxt, PointerGetDatum(NULL),
- jsv, &element_isnull);
+ jsv, &element_isnull, ctx->escontext);
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
accumArrayResult(ctx->astate, element, element_isnull,
ctx->aio->element_type, ctx->acxt);
Assert(ndim > 0);
ctx->sizes[ndim - 1]++; /* increment current dimension counter */
+
+ return true;
}
/* json object start handler for populate_array_json() */
@@ -2592,7 +2630,12 @@ populate_array_object_start(void *_state)
if (state->ctx->ndims <= 0)
populate_array_assign_ndims(state->ctx, ndim);
else if (ndim < state->ctx->ndims)
+ {
populate_array_report_expected_array(state->ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2609,7 +2652,11 @@ populate_array_array_end(void *_state)
populate_array_assign_ndims(ctx, ndim + 1);
if (ndim < ctx->ndims)
- populate_array_check_dimension(ctx, ndim);
+ {
+ /* Report if an error occurred. */
+ if (!populate_array_check_dimension(ctx, ndim))
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2667,7 +2714,9 @@ populate_array_element_end(void *_state, bool isnull)
state->element_start) * sizeof(char);
}
- populate_array_element(ctx, ndim, &jsv);
+ /* Report if an error occurred. */
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return JSON_SEM_ACTION_FAILED;
}
return JSON_SUCCESS;
@@ -2684,7 +2733,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
if (ctx->ndims <= 0)
populate_array_assign_ndims(ctx, ndim);
else if (ndim < ctx->ndims)
+ {
populate_array_report_expected_array(ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
if (ndim == ctx->ndims)
{
@@ -2697,8 +2751,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
return JSON_SUCCESS;
}
-/* parse a json array and populate array */
-static void
+/*
+ * Parse a json array and populate array
+ *
+ * Returns false if an error occurs when parsing.
+ */
+static bool
populate_array_json(PopulateArrayContext *ctx, char *json, int len)
{
PopulateArrayState state;
@@ -2716,19 +2774,25 @@ populate_array_json(PopulateArrayContext *ctx, char *json, int len)
sem.array_element_end = populate_array_element_end;
sem.scalar = populate_array_scalar;
- pg_parse_json_or_ereport(state.lex, &sem);
-
- /* number of dimensions should be already known */
- Assert(ctx->ndims > 0 && ctx->dims);
+ if (pg_parse_json_or_errsave(state.lex, &sem, ctx->escontext))
+ {
+ /* number of dimensions should be already known */
+ Assert(ctx->ndims > 0 && ctx->dims);
+ }
freeJsonLexContext(state.lex);
+
+ return !SOFT_ERROR_OCCURRED(ctx->escontext);
}
/*
* populate_array_dim_jsonb() -- Iterate recursively through jsonb sub-array
* elements and accumulate result using given ArrayBuildState.
+ *
+ * Returns false if we return partway through because of an error in a
+ * subroutine.
*/
-static void
+static bool
populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
JsonbValue *jbv, /* jsonb sub-array */
int ndim) /* current dimension */
@@ -2742,7 +2806,12 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ {
populate_array_report_expected_array(ctx, ndim - 1);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
Assert(!JsonContainerIsScalar(jbc));
@@ -2763,7 +2832,10 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
(tok == WJB_ELEM &&
(val.type != jbvBinary ||
!JsonContainerIsArray(val.val.binary.data)))))
- populate_array_assign_ndims(ctx, ndim);
+ {
+ if (!populate_array_assign_ndims(ctx, ndim))
+ return false;
+ }
jsv.is_json = false;
jsv.val.jsonb = &val;
@@ -2776,16 +2848,21 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
* it is not the innermost dimension.
*/
if (ctx->ndims > 0 && ndim >= ctx->ndims)
- populate_array_element(ctx, ndim, &jsv);
+ {
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return false;
+ }
else
{
/* populate child sub-array */
- populate_array_dim_jsonb(ctx, &val, ndim + 1);
+ if (!populate_array_dim_jsonb(ctx, &val, ndim + 1))
+ return false;
/* number of dimensions should be already known */
Assert(ctx->ndims > 0 && ctx->dims);
- populate_array_check_dimension(ctx, ndim);
+ if (!populate_array_check_dimension(ctx, ndim))
+ return false;
}
tok = JsonbIteratorNext(&it, &val, true);
@@ -2796,14 +2873,22 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
/* free iterator, iterating until WJB_DONE */
tok = JsonbIteratorNext(&it, &val, true);
Assert(tok == WJB_DONE && !it);
+
+ return true;
}
-/* recursively populate an array from json/jsonb */
+/*
+ * Recursively populate an array from json/jsonb
+ *
+ * *isnull is set to true if an error is reported during parsing.
+ */
static Datum
populate_array(ArrayIOData *aio,
const char *colname,
MemoryContext mcxt,
- JsValue *jsv)
+ JsValue *jsv,
+ bool *isnull,
+ Node *escontext)
{
PopulateArrayContext ctx;
Datum result;
@@ -2818,14 +2903,27 @@ populate_array(ArrayIOData *aio,
ctx.ndims = 0; /* unknown yet */
ctx.dims = NULL;
ctx.sizes = NULL;
+ ctx.escontext = escontext;
if (jsv->is_json)
- populate_array_json(&ctx, jsv->val.json.str,
- jsv->val.json.len >= 0 ? jsv->val.json.len
- : strlen(jsv->val.json.str));
+ {
+ /* Return null if an error was found. */
+ if (!populate_array_json(&ctx, jsv->val.json.str,
+ jsv->val.json.len >= 0 ? jsv->val.json.len
+ : strlen(jsv->val.json.str)))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
+ }
else
{
- populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1);
+ /* Return null if an error was found. */
+ if (!populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
ctx.dims[0] = ctx.sizes[0];
}
@@ -2843,11 +2941,16 @@ populate_array(ArrayIOData *aio,
pfree(ctx.sizes);
pfree(lbs);
+ *isnull = false;
return result;
}
-static void
-JsValueToJsObject(JsValue *jsv, JsObject *jso)
+/*
+ * Returns false if an error occurs, provided escontext points to an
+ * ErrorSaveContext.
+ */
+static bool
+JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext)
{
jso->is_json = jsv->is_json;
@@ -2859,7 +2962,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
jsv->val.json.len >= 0
? jsv->val.json.len
: strlen(jsv->val.json.str),
- "populate_composite");
+ "populate_composite",
+ escontext);
}
else
{
@@ -2877,7 +2981,7 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
is_scalar = IsAJsonbScalar(jbv) ||
(jbv->type == jbvBinary &&
JsonContainerIsScalar(jbv->val.binary.data));
- ereport(ERROR,
+ errsave(escontext,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
is_scalar
? errmsg("cannot call %s on a scalar",
@@ -2886,6 +2990,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
"populate_composite")));
}
}
+
+ return !SOFT_ERROR_OCCURRED(escontext);
}
/* acquire or update cached tuple descriptor for a composite type */
@@ -2912,7 +3018,12 @@ update_cached_tupdesc(CompositeIOData *io, MemoryContext mcxt)
}
}
-/* recursively populate a composite (row type) value from json/jsonb */
+/*
+ * Recursively populate a composite (row type) value from json/jsonb
+ *
+ * Returns null if an error occurs in a subroutine, provided escontext points
+ * to an ErrorSaveContext.
+ */
static Datum
populate_composite(CompositeIOData *io,
Oid typid,
@@ -2920,14 +3031,15 @@ populate_composite(CompositeIOData *io,
MemoryContext mcxt,
HeapTupleHeader defaultval,
JsValue *jsv,
- bool isnull)
+ bool *isnull,
+ Node *escontext)
{
Datum result;
/* acquire/update cached tuple descriptor */
update_cached_tupdesc(io, mcxt);
- if (isnull)
+ if (*isnull)
result = (Datum) 0;
else
{
@@ -2935,11 +3047,15 @@ populate_composite(CompositeIOData *io,
JsObject jso;
/* prepare input value */
- JsValueToJsObject(jsv, &jso);
+ if (!JsValueToJsObject(jsv, &jso, escontext))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
/* populate resulting record tuple */
tuple = populate_record(io->tupdesc, &io->record_io,
- defaultval, mcxt, &jso);
+ defaultval, mcxt, &jso, escontext);
result = HeapTupleHeaderGetDatum(tuple);
JsObjectFree(&jso);
@@ -2951,14 +3067,20 @@ populate_composite(CompositeIOData *io,
* now, we can tell by comparing typid to base_typid.)
*/
if (typid != io->base_typid && typid != RECORDOID)
- domain_check(result, isnull, typid, &io->domain_info, mcxt);
+ domain_check(result, *isnull, typid, &io->domain_info, mcxt);
return result;
}
-/* populate non-null scalar value from json/jsonb value */
+/*
+ * Populate non-null scalar value from json/jsonb value.
+ *
+ * Returns null if an error occurs during the call to type input function,
+ * provided escontext is valid.
+ */
static Datum
-populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
+populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext)
{
Datum res;
char *str = NULL;
@@ -3029,7 +3151,12 @@ populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
elog(ERROR, "unrecognized jsonb type: %d", (int) jbv->type);
}
- res = InputFunctionCall(&io->typiofunc, str, io->typioparam, typmod);
+ if (!InputFunctionCallSafe(&io->typiofunc, str, io->typioparam, typmod,
+ escontext, &res))
+ {
+ res = (Datum) 0;
+ *isnull = true;
+ }
/* free temporary buffer */
if (str != json)
@@ -3044,7 +3171,8 @@ populate_domain(DomainIOData *io,
const char *colname,
MemoryContext mcxt,
JsValue *jsv,
- bool isnull)
+ bool isnull,
+ Node *escontext)
{
Datum res;
@@ -3055,8 +3183,8 @@ populate_domain(DomainIOData *io,
res = populate_record_field(io->base_io,
io->base_typid, io->base_typmod,
colname, mcxt, PointerGetDatum(NULL),
- jsv, &isnull);
- Assert(!isnull);
+ jsv, &isnull, escontext);
+ Assert(!isnull || SOFT_ERROR_OCCURRED(escontext));
}
domain_check(res, isnull, typid, &io->domain_info, mcxt);
@@ -3160,7 +3288,8 @@ populate_record_field(ColumnIOData *col,
MemoryContext mcxt,
Datum defaultval,
JsValue *jsv,
- bool *isnull)
+ bool *isnull,
+ Node *escontext)
{
TypeCat typcat;
@@ -3193,10 +3322,12 @@ populate_record_field(ColumnIOData *col,
switch (typcat)
{
case TYPECAT_SCALAR:
- return populate_scalar(&col->scalar_io, typid, typmod, jsv);
+ return populate_scalar(&col->scalar_io, typid, typmod, jsv,
+ isnull, escontext);
case TYPECAT_ARRAY:
- return populate_array(&col->io.array, colname, mcxt, jsv);
+ return populate_array(&col->io.array, colname, mcxt, jsv,
+ isnull, escontext);
case TYPECAT_COMPOSITE:
case TYPECAT_COMPOSITE_DOMAIN:
@@ -3205,11 +3336,12 @@ populate_record_field(ColumnIOData *col,
DatumGetPointer(defaultval)
? DatumGetHeapTupleHeader(defaultval)
: NULL,
- jsv, *isnull);
+ jsv, isnull,
+ escontext);
case TYPECAT_DOMAIN:
return populate_domain(&col->io.domain, typid, colname, mcxt,
- jsv, *isnull);
+ jsv, *isnull, escontext);
default:
elog(ERROR, "unrecognized type category '%c'", typcat);
@@ -3266,7 +3398,8 @@ populate_record(TupleDesc tupdesc,
RecordIOData **record_p,
HeapTupleHeader defaultval,
MemoryContext mcxt,
- JsObject *obj)
+ JsObject *obj,
+ Node *escontext)
{
RecordIOData *record = *record_p;
Datum *values;
@@ -3358,7 +3491,8 @@ populate_record(TupleDesc tupdesc,
mcxt,
nulls[i] ? (Datum) 0 : values[i],
&field,
- &nulls[i]);
+ &nulls[i],
+ escontext);
}
res = heap_form_tuple(tupdesc, values, nulls);
@@ -3445,6 +3579,7 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
JsValue jsv = {0};
HeapTupleHeader rec;
Datum rettuple;
+ bool isnull;
JsonbValue jbv;
MemoryContext fnmcxt = fcinfo->flinfo->fn_mcxt;
PopulateRecordCache *cache = fcinfo->flinfo->fn_extra;
@@ -3531,8 +3666,11 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
jbv.val.binary.len = VARSIZE(jb) - VARHDRSZ;
}
+ isnull = false;
rettuple = populate_composite(&cache->c.io.composite, cache->argtype,
- NULL, fnmcxt, rec, &jsv, false);
+ NULL, fnmcxt, rec, &jsv, &isnull,
+ NULL);
+ Assert(!isnull);
PG_RETURN_DATUM(rettuple);
}
@@ -3543,7 +3681,8 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
* decompose a json object into a hash table.
*/
static HTAB *
-get_json_object_as_hash(char *json, int len, const char *funcname)
+get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext)
{
HASHCTL ctl;
HTAB *tab;
@@ -3572,7 +3711,7 @@ get_json_object_as_hash(char *json, int len, const char *funcname)
sem->object_field_start = hash_object_field_start;
sem->object_field_end = hash_object_field_end;
- pg_parse_json_or_ereport(state->lex, sem);
+ pg_parse_json_or_errsave(state->lex, sem, escontext);
freeJsonLexContext(state->lex);
@@ -3743,7 +3882,8 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
&cache->c.io.composite.record_io,
state->rec,
cache->fn_mcxt,
- obj);
+ obj,
+ NULL);
/* if it's domain over composite, check domain constraints */
if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
--
2.35.3
[application/octet-stream] v24-0004-JSON_TABLE.patch (165.3K, ../../CA+HiwqG_XFksg7HN_gbmcOQU00Q5bvTKeLSZ3NftewdBRw4fKA@mail.gmail.com/5-v24-0004-JSON_TABLE.patch)
download | inline diff:
From 9550253853c1a9f3412fcec415b758d46b960247 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:14 +0900
Subject: [PATCH v24 4/5] JSON_TABLE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This feature allows jsonb data to be treated as a table and thus
used in a FROM clause like other tabular data. Data can be selected
from the jsonb using jsonpath expressions, and hoisted out of nested
structures in the jsonb to form multiple rows, more or less like an
outer join.
This also adds the PLAN clauses for JSON_TABLE, which allow the user
to specify how data from nested paths are joined, allowing
considerable freedom in shaping the tabular output of JSON_TABLE.
PLAN DEFAULT allows the user to specify the global strategies when
dealing with sibling or child nested paths. The is often sufficient
to achieve the necessary goal, and is considerably simpler than the
full PLAN clause, which allows the user to specify the strategy to be
used for each named nested path.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 496 ++++++++
src/backend/commands/explain.c | 8 +-
src/backend/executor/execExprInterp.c | 6 +
src/backend/executor/nodeTableFuncscan.c | 27 +-
src/backend/nodes/makefuncs.c | 19 +
src/backend/nodes/nodeFuncs.c | 28 +
src/backend/parser/Makefile | 1 +
src/backend/parser/gram.y | 312 ++++-
src/backend/parser/meson.build | 1 +
src/backend/parser/parse_clause.c | 14 +-
src/backend/parser/parse_expr.c | 13 +
src/backend/parser/parse_jsontable.c | 760 ++++++++++++
src/backend/parser/parse_relation.c | 5 +-
src/backend/parser/parse_target.c | 3 +
src/backend/utils/adt/jsonpath_exec.c | 543 +++++++++
src/backend/utils/adt/ruleutils.c | 279 ++++-
src/include/nodes/execnodes.h | 2 +
src/include/nodes/makefuncs.h | 2 +
src/include/nodes/parsenodes.h | 89 ++
src/include/nodes/primnodes.h | 61 +-
src/include/parser/kwlist.h | 4 +
src/include/parser/parse_clause.h | 3 +
src/include/utils/jsonpath.h | 3 +
src/test/regress/expected/json_sqljson.out | 6 +
src/test/regress/expected/jsonb_sqljson.out | 1186 +++++++++++++++++++
src/test/regress/sql/json_sqljson.sql | 4 +
src/test/regress/sql/jsonb_sqljson.sql | 674 +++++++++++
src/tools/pgindent/typedefs.list | 13 +
28 files changed, 4535 insertions(+), 27 deletions(-)
create mode 100644 src/backend/parser/parse_jsontable.c
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index c12ee06713..39584fba58 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17214,6 +17214,502 @@ array w/o UK? | t
</table>
</sect2>
+ <sect2 id="functions-sqljson-table">
+ <title>JSON_TABLE</title>
+ <indexterm>
+ <primary>json_table</primary>
+ </indexterm>
+
+ <para>
+ <function>json_table</function> is an SQL/JSON function which
+ queries <acronym>JSON</acronym> data
+ and presents the results as a relational view, which can be accessed as a
+ regular SQL table. You can only use <function>json_table</function> inside the
+ <literal>FROM</literal> clause of a <literal>SELECT</literal> statement.
+ </para>
+
+ <para>
+ Taking JSON data as input, <function>json_table</function> uses
+ a path expression to extract a part of the provided data that
+ will be used as a <firstterm>row pattern</firstterm> for the
+ constructed view. Each SQL/JSON item at the top level of the row pattern serves
+ as the source for a separate row in the constructed relational view.
+ </para>
+
+ <para>
+ To split the row pattern into columns, <function>json_table</function>
+ provides the <literal>COLUMNS</literal> clause that defines the
+ schema of the created view. For each column to be constructed,
+ this clause provides a separate path expression that evaluates
+ the row pattern, extracts a JSON item, and returns it as a
+ separate SQL value for the specified column. If the required value
+ is stored in a nested level of the row pattern, it can be extracted
+ using the <literal>NESTED PATH</literal> subclause. Joining the
+ columns returned by <literal>NESTED PATH</literal> can add multiple
+ new rows to the constructed view. Such rows are called
+ <firstterm>child rows</firstterm>, as opposed to the <firstterm>parent row</firstterm>
+ that generates them.
+ </para>
+
+ <para>
+ The rows produced by <function>JSON_TABLE</function> are laterally
+ joined to the row that generated them, so you do not have to explicitly join
+ the constructed view with the original table holding <acronym>JSON</acronym>
+ data. Optionally, you can specify how to join the columns returned
+ by <literal>NESTED PATH</literal> using the <literal>PLAN</literal> clause.
+ </para>
+
+ <para>
+ Each <literal>NESTED PATH</literal> clause can generate one or more
+ columns. Columns produced by <literal>NESTED PATH</literal>s at the
+ same level are considered to be <firstterm>siblings</firstterm>,
+ while a column produced by a <literal>NESTED PATH</literal> is
+ considered to be a child of the column produced by a
+ <literal>NESTED PATH</literal> or row expression at a higher level.
+ Sibling columns are always joined first. Once they are processed,
+ the resulting rows are joined to the parent row.
+ </para>
+
+ <para>
+ The syntax is:
+ </para>
+
+<synopsis>
+JSON_TABLE (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> AS <replaceable>json_path_name</replaceable> </optional> <optional> PASSING { <replaceable>value</replaceable> AS <replaceable>varname</replaceable> } <optional>, ...</optional> </optional>
+ COLUMNS ( <replaceable class="parameter">json_table_column</replaceable> <optional>, ...</optional> )
+ <optional> { <literal>ERROR</literal> | <literal>EMPTY</literal> } <literal>ON ERROR</literal> </optional>
+ <optional>
+ PLAN ( <replaceable class="parameter">json_table_plan</replaceable> ) |
+ PLAN DEFAULT ( { INNER | OUTER } <optional> , { CROSS | UNION } </optional>
+ | { CROSS | UNION } <optional> , { INNER | OUTER } </optional> )
+ </optional>
+)
+
+<phrase>
+where <replaceable class="parameter">json_table_column</replaceable> is:
+</phrase>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> FORMAT <replaceable>json_representation</replaceable>
+ <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> EXISTS <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { ERROR | TRUE | FALSE | UNKNOWN } ON ERROR </optional>
+ | NESTED PATH <replaceable>json_path_specification</replaceable> <optional> AS <replaceable>path_name</replaceable> </optional>
+ COLUMNS ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ | <replaceable>name</replaceable> FOR ORDINALITY
+<phrase>
+<replaceable>json_table_plan</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> <optional> { OUTER | INNER } <replaceable>json_table_plan_primary</replaceable> </optional>
+ | <replaceable>json_table_plan_primary</replaceable> { UNION <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+ | <replaceable>json_table_plan_primary</replaceable> { CROSS <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+<phrase>
+<replaceable>json_table_plan_primary</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> | ( <replaceable>json_table_plan</replaceable> )
+</synopsis>
+
+ <para>
+ Each syntax element is described below in more detail.
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional></literal>
+ </term>
+ <listitem>
+ <para>
+ The input data to query, the JSON path expression defining the query,
+ and an optional <literal>PASSING</literal> clause, which can provide data
+ values to the <replaceable>path_expression</replaceable>.
+ The result of the input data
+ evaluation is called the <firstterm>row pattern</firstterm>. The row
+ pattern is used as the source for row values in the constructed view.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>COLUMNS</literal>( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ The <literal>COLUMNS</literal> clause defining the schema of the
+ constructed view. In this clause, you must specify all the columns
+ to be filled with SQL/JSON items.
+ The <replaceable>json_table_column</replaceable>
+ expression has the following syntax variants:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>name</replaceable> <replaceable>type</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional></literal>
+ </term>
+ <listitem>
+
+ <para>
+ Inserts a single SQL/JSON item into each row of
+ the specified column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON EMPTY</literal> and
+ <literal>ON ERROR</literal> clauses to define how to handle missing values
+ or structural errors.
+ <literal>WRAPPER</literal> and <literal>QUOTES</literal> clauses can only
+ be used with JSON, array, and composite types.
+ These clauses have the same syntax and semantics as for
+ <function>json_value</function> and <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <literal>FORMAT</literal> <replaceable>json_representation</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a composite SQL/JSON
+ item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>WRAPPER</literal>, <literal>QUOTES</literal>,
+ <literal>ON EMPTY</literal> and <literal>ON ERROR</literal> clauses
+ to define additional settings for the returned SQL/JSON items.
+ These clauses have the same syntax and semantics as
+ for <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable>
+ <literal>EXISTS</literal> <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a boolean item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated,
+ a check whether any SQL/JSON items were returned is done, and
+ the column is filled with the resulting boolean value, one for each row.
+ The specified <parameter>type</parameter> should have a cast from the
+ <type>boolean</type>.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON ERROR</literal> clause to define
+ error behavior. This clause has the same syntax and semantics as
+ for <function>json_exists</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>NESTED PATH</literal> <replaceable>json_path_specification</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional>
+ <literal>COLUMNS</literal> ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ Extracts SQL/JSON items from nested levels of the row pattern,
+ generates one or more columns as defined by the <literal>COLUMNS</literal>
+ subclause, and inserts the extracted SQL/JSON items into each row of these columns.
+ The <replaceable>json_table_column</replaceable> expression in the
+ <literal>COLUMNS</literal> subclause uses the same syntax as in the
+ parent <literal>COLUMNS</literal> clause.
+ </para>
+
+ <para>
+ The <literal>NESTED PATH</literal> syntax is recursive,
+ so you can go down multiple nested levels by specifying several
+ <literal>NESTED PATH</literal> subclauses within each other.
+ It allows to unnest the hierarchy of JSON objects and arrays
+ in a single function invocation rather than chaining several
+ <function>JSON_TABLE</function> expressions in an SQL statement.
+ </para>
+
+ <para>
+ You can use the <literal>PLAN</literal> clause to define how
+ to join the columns returned by <literal>NESTED PATH</literal> clauses.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <literal>FOR ORDINALITY</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Adds an ordinality column that provides sequential row numbering.
+ You can have only one ordinality column per table. Row numbering
+ is 1-based. For child rows that result from the <literal>NESTED PATH</literal>
+ clauses, the parent row number is repeated.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>AS</literal> <replaceable>json_path_name</replaceable>
+ </term>
+ <listitem>
+
+ <para>
+ The optional <replaceable>json_path_name</replaceable> serves as an
+ identifier of the provided <replaceable>json_path_specification</replaceable>.
+ The path name must be unique and distinct from the column names.
+ When using the <literal>PLAN</literal> clause, you must specify the names
+ for all the paths, including the row pattern. Each path name can appear in
+ the <literal>PLAN</literal> clause only once.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN</literal> ( <replaceable>json_table_plan</replaceable> )
+ </term>
+ <listitem>
+
+ <para>
+ Defines how to join the data returned by <literal>NESTED PATH</literal>
+ clauses to the constructed view.
+ </para>
+ <para>
+ To join columns with parent/child relationship, you can use:
+ </para>
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>INNER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>INNER JOIN</literal>, so that the parent row
+ is omitted from the output if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>OUTER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>LEFT OUTER JOIN</literal>, so that the parent row
+ is always included into the output even if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>, with NULL values
+ inserted into the child columns if the corresponding
+ values are missing.
+ </para>
+ <para>
+ This is the default option for joining columns with parent/child relationship.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>
+ To join sibling columns, you can use:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>UNION</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each value produced by each of the sibling
+ columns. The columns from the other siblings are set to null.
+ </para>
+ <para>
+ This is the default option for joining sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>CROSS</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each combination of values from the sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN DEFAULT</literal> ( <literal><replaceable>OUTER | INNER</replaceable> <optional>, <replaceable>UNION | CROSS</replaceable> </optional></literal> )
+ </term>
+ <listitem>
+ <para>
+ The terms can also be specified in reverse order. The
+ <literal>INNER</literal> or <literal>OUTER</literal> option defines the
+ joining plan for parent/child columns, while <literal>UNION</literal> or
+ <literal>CROSS</literal> affects joins of sibling columns. This form
+ of <literal>PLAN</literal> overrides the default plan for
+ all columns at once. Even though the path names are not included in the
+ <literal>PLAN DEFAULT</literal> form, to conform to the SQL/JSON standard
+ they must be provided for all the paths if the <literal>PLAN</literal>
+ clause is used.
+ </para>
+ <para>
+ <literal>PLAN DEFAULT</literal> is simpler than specifying a complete
+ <literal>PLAN</literal>, and is often all that is required to get the desired
+ output.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>Examples</para>
+
+ <para>
+ In these examples the following small table storing some JSON data will be used:
+<programlisting>
+CREATE TABLE my_films ( js jsonb );
+
+INSERT INTO my_films VALUES (
+'{ "favorites" : [
+ { "kind" : "comedy", "films" : [
+ { "title" : "Bananas",
+ "director" : "Woody Allen"},
+ { "title" : "The Dinner Game",
+ "director" : "Francis Veber" } ] },
+ { "kind" : "horror", "films" : [
+ { "title" : "Psycho",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "thriller", "films" : [
+ { "title" : "Vertigo",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "drama", "films" : [
+ { "title" : "Yojimbo",
+ "director" : "Akira Kurosawa" } ] }
+ ] }');
+</programlisting>
+ </para>
+ <para>
+ Query the <structname>my_films</structname> table holding
+ some JSON data about the films and create a view that
+ distributes the film genre, title, and director between separate columns:
+<screen>
+SELECT jt.* FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites[*]' COLUMNS (
+ id FOR ORDINALITY,
+ kind text PATH '$.kind',
+ NESTED PATH '$.films[*]' COLUMNS (
+ title text PATH '$.title',
+ director text PATH '$.director'))) AS jt;
+----+----------+------------------+-------------------
+ id | kind | title | director
+----+----------+------------------+-------------------
+ 1 | comedy | Bananas | Woody Allen
+ 1 | comedy | The Dinner Game | Francis Veber
+ 2 | horror | Psycho | Alfred Hitchcock
+ 3 | thriller | Vertigo | Alfred Hitchcock
+ 4 | drama | Yojimbo | Akira Kurosawa
+ (5 rows)
+</screen>
+ </para>
+
+ <para>
+ Find a director that has done films in two different genres:
+<screen>
+SELECT
+ director1 AS director, title1, kind1, title2, kind2
+FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites' AS favs COLUMNS (
+ NESTED PATH '$[*]' AS films1 COLUMNS (
+ kind1 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film1 COLUMNS (
+ title1 text PATH '$.title',
+ director1 text PATH '$.director')
+ ),
+ NESTED PATH '$[*]' AS films2 COLUMNS (
+ kind2 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film2 COLUMNS (
+ title2 text PATH '$.title',
+ director2 text PATH '$.director'
+ )
+ )
+ )
+ PLAN (favs OUTER ((films1 INNER film1) CROSS (films2 INNER film2)))
+ ) AS jt
+ WHERE kind1 > kind2 AND director1 = director2;
+
+ director | title1 | kind1 | title2 | kind2
+------------------+---------+----------+--------+--------
+ Alfred Hitchcock | Vertigo | thriller | Psycho | horror
+(1 row)
+</screen>
+ </para>
+ </sect2>
+
<sect2 id="functions-sqljson-path">
<title>The SQL/JSON Path Language</title>
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index f1d71bc54e..8e35525781 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -3892,7 +3892,13 @@ ExplainTargetRel(Plan *plan, Index rti, ExplainState *es)
break;
case T_TableFuncScan:
Assert(rte->rtekind == RTE_TABLEFUNC);
- objectname = "xmltable";
+ if (rte->tablefunc)
+ if (rte->tablefunc->functype == TFT_XMLTABLE)
+ objectname = "xmltable";
+ else /* Must be TFT_JSON_TABLE */
+ objectname = "json_table";
+ else
+ objectname = NULL;
objecttag = "Table Function Name";
break;
case T_ValuesScan:
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 92e15fdc7f..666977d09c 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -4364,6 +4364,12 @@ ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
break;
}
+ case JSON_TABLE_OP:
+ post_eval->jump_eval_coercion = -1;
+ *op->resvalue = item;
+ *op->resnull = false;
+ break;
+
default:
elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
return false;
diff --git a/src/backend/executor/nodeTableFuncscan.c b/src/backend/executor/nodeTableFuncscan.c
index a60dcd4943..0d7f518afd 100644
--- a/src/backend/executor/nodeTableFuncscan.c
+++ b/src/backend/executor/nodeTableFuncscan.c
@@ -28,6 +28,7 @@
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "utils/builtins.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/xml.h"
@@ -161,8 +162,9 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
scanstate->ss.ps.qual =
ExecInitQual(node->scan.plan.qual, &scanstate->ss.ps);
- /* Only XMLTABLE is supported currently */
- scanstate->routine = &XmlTableRoutine;
+ /* Only XMLTABLE and JSON_TABLE are supported currently */
+ scanstate->routine =
+ tf->functype == TFT_XMLTABLE ? &XmlTableRoutine : &JsonbTableRoutine;
scanstate->perTableCxt =
AllocSetContextCreate(CurrentMemoryContext,
@@ -182,6 +184,10 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
ExecInitExprList(tf->colexprs, (PlanState *) scanstate);
scanstate->coldefexprs =
ExecInitExprList(tf->coldefexprs, (PlanState *) scanstate);
+ scanstate->colvalexprs =
+ ExecInitExprList(tf->colvalexprs, (PlanState *) scanstate);
+ scanstate->passingvalexprs =
+ ExecInitExprList(tf->passingvalexprs, (PlanState *) scanstate);
scanstate->notnulls = tf->notnulls;
@@ -369,14 +375,17 @@ tfuncInitialize(TableFuncScanState *tstate, ExprContext *econtext, Datum doc)
routine->SetNamespace(tstate, ns_name, ns_uri);
}
- /* Install the row filter expression into the table builder context */
- value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
- if (isnull)
- ereport(ERROR,
- (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
- errmsg("row filter expression must not be null")));
+ if (routine->SetRowFilter)
+ {
+ /* Install the row filter expression into the table builder context */
+ value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
+ if (isnull)
+ ereport(ERROR,
+ (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
+ errmsg("row filter expression must not be null")));
- routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ }
/*
* Install the column filter expressions into the table builder context.
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index 746ed3d1da..2ad7b8cb7b 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -890,6 +890,25 @@ makeJsonBehavior(JsonBehaviorType type, Node *default_expr, int location)
return behavior;
}
+/*
+ * makeJsonTableJoinedPlan -
+ * creates a joined JsonTablePlan node
+ */
+Node *
+makeJsonTableJoinedPlan(JsonTablePlanJoinType type, Node *plan1, Node *plan2,
+ int location)
+{
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_JOINED;
+ n->join_type = type;
+ n->plan1 = castNode(JsonTablePlan, plan1);
+ n->plan2 = castNode(JsonTablePlan, plan2);
+ n->location = location;
+
+ return (Node *) n;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 8e49ef6b95..df951c212c 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -2625,6 +2625,10 @@ expression_tree_walker_impl(Node *node,
return true;
if (WALK(tf->coldefexprs))
return true;
+ if (WALK(tf->colvalexprs))
+ return true;
+ if (WALK(tf->passingvalexprs))
+ return true;
}
break;
default:
@@ -3685,6 +3689,8 @@ expression_tree_mutator_impl(Node *node,
MUTATE(newnode->rowexpr, tf->rowexpr, Node *);
MUTATE(newnode->colexprs, tf->colexprs, List *);
MUTATE(newnode->coldefexprs, tf->coldefexprs, List *);
+ MUTATE(newnode->colvalexprs, tf->colvalexprs, List *);
+ MUTATE(newnode->passingvalexprs, tf->passingvalexprs, List *);
return (Node *) newnode;
}
break;
@@ -4124,6 +4130,28 @@ raw_expression_tree_walker_impl(Node *node,
return true;
}
break;
+ case T_JsonTable:
+ {
+ JsonTable *jt = (JsonTable *) node;
+
+ if (WALK(jt->common))
+ return true;
+ if (WALK(jt->columns))
+ return true;
+ }
+ break;
+ case T_JsonTableColumn:
+ {
+ JsonTableColumn *jtc = (JsonTableColumn *) node;
+
+ if (WALK(jtc->typeName))
+ return true;
+ if (WALK(jtc->behavior))
+ return true;
+ if (jtc->coltype == JTC_NESTED && WALK(jtc->columns))
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/parser/Makefile b/src/backend/parser/Makefile
index 401c16686c..3162a01f30 100644
--- a/src/backend/parser/Makefile
+++ b/src/backend/parser/Makefile
@@ -23,6 +23,7 @@ OBJS = \
parse_enr.o \
parse_expr.o \
parse_func.o \
+ parse_jsontable.o \
parse_merge.o \
parse_node.o \
parse_oper.o \
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 1dc3300fde..56895cf47d 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -655,13 +655,36 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_aggregate_func
json_api_common_syntax
json_argument
+ json_table
+ json_table_column_definition
+ json_table_ordinality_column_definition
+ json_table_regular_column_definition
+ json_table_formatted_column_definition
+ json_table_exists_column_definition
+ json_table_nested_columns
+ json_table_plan_clause_opt
+ json_table_plan
+ json_table_plan_simple
+ json_table_plan_parent_child
+ json_table_plan_outer
+ json_table_plan_inner
+ json_table_plan_sibling
+ json_table_plan_union
+ json_table_plan_cross
+ json_table_plan_primary
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
json_arguments
+ json_table_columns_clause
+ json_table_column_definition_list
+%type <str> json_table_column_path_specification_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
+ json_table_default_plan_choices
+ json_table_default_plan_inner_outer
+ json_table_default_plan_union_cross
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
@@ -733,7 +756,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
- JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_TABLE JSON_VALUE
KEY KEYS KEEP
@@ -744,8 +767,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
- NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
- NORMALIZE NORMALIZED
+ NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
+ NONE NORMALIZE NORMALIZED
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
@@ -753,8 +776,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PATH
+ PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -862,6 +885,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
* Using the same precedence as IDENT seems right for the reasons given above.
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
+%nonassoc COLUMNS
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
@@ -884,6 +908,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%left JOIN CROSS LEFT FULL RIGHT INNER_P NATURAL
+%nonassoc json_table_column
+%nonassoc NESTED
+%left PATH
%%
/*
@@ -13410,6 +13437,21 @@ table_ref: relation_expr opt_alias_clause
$2->alias = $4;
$$ = (Node *) $2;
}
+ | json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $1);
+
+ jt->alias = $2;
+ $$ = (Node *) jt;
+ }
+ | LATERAL_P json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $2);
+
+ jt->alias = $3;
+ jt->lateral = true;
+ $$ = (Node *) jt;
+ }
;
@@ -13977,6 +14019,8 @@ xmltable_column_option_el:
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(true), @1); }
| NULL_P
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(false), @1); }
+ | PATH b_expr
+ { $$ = makeDefElem("path", $2, @1); }
;
xml_namespace_list:
@@ -16657,6 +16701,256 @@ json_quotes_clause_opt:
| /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
;
+json_table:
+ JSON_TABLE '('
+ json_api_common_syntax
+ json_table_columns_clause
+ json_table_plan_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonTable *n = makeNode(JsonTable);
+
+ n->common = (JsonCommon *) $3;
+ n->columns = $4;
+ n->plan = (JsonTablePlan *) $5;
+ n->behavior = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_columns_clause:
+ COLUMNS '(' json_table_column_definition_list ')' { $$ = $3; }
+ ;
+
+json_table_column_definition_list:
+ json_table_column_definition
+ { $$ = list_make1($1); }
+ | json_table_column_definition_list ',' json_table_column_definition
+ { $$ = lappend($1, $3); }
+ ;
+
+json_table_column_definition:
+ json_table_ordinality_column_definition %prec json_table_column
+ | json_table_regular_column_definition %prec json_table_column
+ | json_table_formatted_column_definition %prec json_table_column
+ | json_table_exists_column_definition %prec json_table_column
+ | json_table_nested_columns
+ ;
+
+json_table_ordinality_column_definition:
+ ColId FOR ORDINALITY
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FOR_ORDINALITY;
+ n->name = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_regular_column_definition:
+ ColId Typename
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_REGULAR;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->pathspec = $3;
+ n->wrapper = $4; /* disallowed during parse-analysis! */
+ n->quotes = $5; /* disallowed during parse-analysis! */
+ n->behavior = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_formatted_column_definition:
+ ColId Typename FORMAT_LA JSON json_encoding_clause_opt
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FORMATTED;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_JSON, $5, @1);
+ n->pathspec = $6;
+ n->wrapper = $7;
+ if (n->wrapper != JSW_NONE && $8 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@7)));
+ n->quotes = $8;
+ n->behavior = $9;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_exists_column_definition:
+ ColId Typename
+ EXISTS json_table_column_path_specification_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_EXISTS;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = JSW_NONE;
+ n->quotes = JS_QUOTES_UNSPEC;
+ n->pathspec = $4;
+ n->behavior = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_column_path_specification_clause_opt:
+ PATH Sconst { $$ = $2; }
+ | /* EMPTY */ %prec json_table_column { $$ = NULL; }
+ ;
+
+json_table_nested_columns:
+ NESTED path_opt Sconst
+ json_table_columns_clause
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->columns = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | NESTED path_opt Sconst AS name
+ json_table_columns_clause
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->columns = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+path_opt:
+ PATH { }
+ | /* EMPTY */ { }
+ ;
+
+json_table_plan_clause_opt:
+ PLAN '(' json_table_plan ')' { $$ = $3; }
+ | PLAN DEFAULT '(' json_table_default_plan_choices ')'
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_DEFAULT;
+ n->join_type = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | /* EMPTY */ { $$ = NULL; }
+ ;
+
+json_table_plan:
+ json_table_plan_simple
+ | json_table_plan_parent_child
+ | json_table_plan_sibling
+ ;
+
+json_table_plan_simple:
+ name
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_SIMPLE;
+ n->pathname = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_plan_primary:
+ json_table_plan_simple { $$ = $1; }
+ | '(' json_table_plan ')'
+ {
+ castNode(JsonTablePlan, $2)->location = @1;
+ $$ = $2;
+ }
+ ;
+
+json_table_plan_outer:
+ json_table_plan_simple OUTER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_OUTER, $1, $3, @1); }
+ ;
+
+json_table_plan_inner:
+ json_table_plan_simple INNER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_INNER, $1, $3, @1); }
+ ;
+
+json_table_plan_parent_child:
+ json_table_plan_outer
+ | json_table_plan_inner
+ ;
+
+json_table_plan_union:
+ json_table_plan_primary UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ | json_table_plan_union UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ ;
+
+json_table_plan_cross:
+ json_table_plan_primary CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ | json_table_plan_cross CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ ;
+
+json_table_plan_sibling:
+ json_table_plan_union
+ | json_table_plan_cross
+ ;
+
+json_table_default_plan_choices:
+ json_table_default_plan_inner_outer { $$ = $1 | JSTPJ_UNION; }
+ | json_table_default_plan_union_cross { $$ = $1 | JSTPJ_OUTER; }
+ | json_table_default_plan_inner_outer ','
+ json_table_default_plan_union_cross { $$ = $1 | $3; }
+ | json_table_default_plan_union_cross ','
+ json_table_default_plan_inner_outer { $$ = $1 | $3; }
+ ;
+
+json_table_default_plan_inner_outer:
+ INNER_P { $$ = JSTPJ_INNER; }
+ | OUTER_P { $$ = JSTPJ_OUTER; }
+ ;
+
+json_table_default_plan_union_cross:
+ UNION { $$ = JSTPJ_UNION; }
+ | CROSS { $$ = JSTPJ_CROSS; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -17405,6 +17699,7 @@ unreserved_keyword:
| MOVE
| NAME_P
| NAMES
+ | NESTED
| NEW
| NEXT
| NFC
@@ -17439,6 +17734,8 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
+ | PLAN
| PLANS
| POLICY
| PRECEDING
@@ -17603,6 +17900,7 @@ col_name_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| LEAST
| NATIONAL
@@ -17971,6 +18269,7 @@ bare_label_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| KEEP
| KEY
@@ -18010,6 +18309,7 @@ bare_label_keyword:
| NATIONAL
| NATURAL
| NCHAR
+ | NESTED
| NEW
| NEXT
| NFC
@@ -18054,7 +18354,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
| PLACING
+ | PLAN
| PLANS
| POLICY
| POSITION
diff --git a/src/backend/parser/meson.build b/src/backend/parser/meson.build
index 94517a7f7d..0f0e4ac7ca 100644
--- a/src/backend/parser/meson.build
+++ b/src/backend/parser/meson.build
@@ -10,6 +10,7 @@ backend_sources += files(
'parse_enr.c',
'parse_expr.c',
'parse_func.c',
+ 'parse_jsontable.c',
'parse_merge.c',
'parse_node.c',
'parse_oper.c',
diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c
index 334b9b42bd..affd812619 100644
--- a/src/backend/parser/parse_clause.c
+++ b/src/backend/parser/parse_clause.c
@@ -697,7 +697,11 @@ transformRangeTableFunc(ParseState *pstate, RangeTableFunc *rtf)
char **names;
int colno;
- /* Currently only XMLTABLE is supported */
+ /*
+ * Currently we only support XMLTABLE here. See transformJsonTable() for
+ * JSON_TABLE support.
+ */
+ tf->functype = TFT_XMLTABLE;
constructName = "XMLTABLE";
docType = XMLOID;
@@ -1104,13 +1108,17 @@ transformFromClauseItem(ParseState *pstate, Node *n,
rtr->rtindex = nsitem->p_rtindex;
return (Node *) rtr;
}
- else if (IsA(n, RangeTableFunc))
+ else if (IsA(n, RangeTableFunc) || IsA(n, JsonTable))
{
/* table function is like a plain relation */
RangeTblRef *rtr;
ParseNamespaceItem *nsitem;
- nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ if (IsA(n, RangeTableFunc))
+ nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ else
+ nsitem = transformJsonTable(pstate, (JsonTable *) n);
+
*top_nsitem = nsitem;
*namespace = list_make1(nsitem);
rtr = makeNode(RangeTblRef);
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64571562c5..11d0f5f5c9 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4351,6 +4351,19 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
JSON_BEHAVIOR_NULL,
jsexpr->returning);
break;
+
+ case JSON_TABLE_OP:
+ func_name = "JSON_TABLE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->returning = makeNode(JsonReturning);
+ jsexpr->returning->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ jsexpr->returning->typid = exprType(jsexpr->formatted_expr);
+ jsexpr->returning->typmod = -1;
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_EMPTY,
+ jsexpr->returning);
+ break;
}
Assert(jsexpr != NULL && jsexpr->formatted_expr != NULL);
diff --git a/src/backend/parser/parse_jsontable.c b/src/backend/parser/parse_jsontable.c
new file mode 100644
index 0000000000..36d888b301
--- /dev/null
+++ b/src/backend/parser/parse_jsontable.c
@@ -0,0 +1,760 @@
+/*-------------------------------------------------------------------------
+ *
+ * parse_jsontable.c
+ * parsing of JSON_TABLE
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/parser/parse_jsontable.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "catalog/pg_collation.h"
+#include "catalog/pg_type.h"
+#include "miscadmin.h"
+#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
+#include "optimizer/optimizer.h"
+#include "parser/parse_clause.h"
+#include "parser/parse_collate.h"
+#include "parser/parse_expr.h"
+#include "parser/parse_relation.h"
+#include "parser/parse_type.h"
+#include "utils/builtins.h"
+#include "utils/json.h"
+#include "utils/lsyscache.h"
+
+/* Context for JSON_TABLE transformation */
+typedef struct JsonTableParseContext
+{
+ ParseState *pstate; /* parsing state */
+ JsonTable *table; /* untransformed node */
+ TableFunc *tablefunc; /* transformed node */
+ List *pathNames; /* list of all path and columns names */
+ int pathNameId; /* path name id counter */
+ Oid contextItemTypid; /* type oid of context item (json/jsonb) */
+} JsonTableParseContext;
+
+static JsonTableParent *transformJsonTableColumns(JsonTableParseContext * cxt,
+ JsonTablePlan *plan,
+ List *columns,
+ char *pathSpec,
+ char **pathName,
+ int location);
+
+static Node *
+makeStringConst(char *str, int location)
+{
+ A_Const *n = makeNode(A_Const);
+
+ n->val.node.type = T_String;
+ n->val.sval.sval = str;
+ n->location = location;
+
+ return (Node *) n;
+}
+
+/*
+ * Transform JSON_TABLE column
+ * - regular column into JSON_VALUE()
+ * - FORMAT JSON column into JSON_QUERY()
+ * - EXISTS column into JSON_EXISTS()
+ */
+static Node *
+transformJsonTableColumn(JsonTableColumn *jtc, Node *contextItemExpr,
+ List *passingArgs, bool errorOnError)
+{
+ JsonFuncExpr *jfexpr = makeNode(JsonFuncExpr);
+ JsonCommon *common = makeNode(JsonCommon);
+ JsonOutput *output = makeNode(JsonOutput);
+ char *pathspec;
+ JsonFormat *default_format;
+
+ jfexpr->op =
+ jtc->coltype == JTC_REGULAR ? JSON_VALUE_OP :
+ jtc->coltype == JTC_EXISTS ? JSON_EXISTS_OP : JSON_QUERY_OP;
+ jfexpr->common = common;
+ jfexpr->output = output;
+ jfexpr->behavior = jtc->behavior;
+ if ((jfexpr->behavior == NULL || jfexpr->behavior->on_error == NULL) &&
+ errorOnError)
+ {
+ JsonBehavior *on_error = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL,
+ -1);
+
+ if (jfexpr->behavior == NULL)
+ jfexpr->behavior = makeJsonBehaviorClause(NULL, on_error, -1);
+ else
+ jfexpr->behavior->on_error = on_error;
+ }
+ jfexpr->quotes = jtc->quotes;
+ jfexpr->wrapper = jtc->wrapper;
+ jfexpr->location = jtc->location;
+
+ output->typeName = jtc->typeName;
+ output->returning = makeNode(JsonReturning);
+ output->returning->format = jtc->format;
+
+ default_format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+
+ common->pathname = NULL;
+ /* formatted_expr will be set later in transformJsonValueExpr(). */
+ common->expr = makeJsonValueExpr((Expr *) contextItemExpr, NULL,
+ default_format);
+ common->passing = passingArgs;
+
+ if (jtc->pathspec)
+ pathspec = jtc->pathspec;
+ else
+ {
+ /* Construct default path as '$."column_name"' */
+ StringInfoData path;
+
+ initStringInfo(&path);
+
+ appendStringInfoString(&path, "$.");
+ escape_json(&path, jtc->name);
+
+ pathspec = path.data;
+ }
+
+ common->pathspec = makeStringConst(pathspec, -1);
+
+ return (Node *) jfexpr;
+}
+
+static bool
+isJsonTablePathNameDuplicate(JsonTableParseContext * cxt, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, cxt->pathNames)
+ {
+ if (!strcmp(pathname, (const char *) lfirst(lc)))
+ return true;
+ }
+
+ return false;
+}
+
+/* Register the column name in the path name list. */
+static void
+registerJsonTableColumn(JsonTableParseContext * cxt, char *colname)
+{
+ if (isJsonTablePathNameDuplicate(cxt, colname))
+ ereport(ERROR,
+ (errcode(ERRCODE_DUPLICATE_ALIAS),
+ errmsg("duplicate JSON_TABLE column name: %s", colname),
+ errhint("JSON_TABLE column names must be distinct from one another.")));
+
+ cxt->pathNames = lappend(cxt->pathNames, colname);
+}
+
+/* Recursively register all nested column names in the path name list. */
+static void
+registerAllJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ if (jtc->pathname)
+ registerJsonTableColumn(cxt, jtc->pathname);
+
+ registerAllJsonTableColumns(cxt, jtc->columns);
+ }
+ else
+ {
+ registerJsonTableColumn(cxt, jtc->name);
+ }
+ }
+}
+
+/* Generate a new unique JSON_TABLE path name. */
+static char *
+generateJsonTablePathName(JsonTableParseContext * cxt)
+{
+ char namebuf[32];
+ char *name = namebuf;
+
+ do
+ {
+ snprintf(namebuf, sizeof(namebuf), "json_table_path_%d",
+ ++cxt->pathNameId);
+ } while (isJsonTablePathNameDuplicate(cxt, name));
+
+ name = pstrdup(name);
+ cxt->pathNames = lappend(cxt->pathNames, name);
+
+ return name;
+}
+
+/* Collect sibling path names from plan to the specified list. */
+static void
+collectSiblingPathsInJsonTablePlan(JsonTablePlan *plan, List **paths)
+{
+ if (plan->plan_type == JSTP_SIMPLE)
+ *paths = lappend(*paths, plan->pathname);
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ *paths = lappend(*paths, plan->plan1->pathname);
+ }
+ else if (plan->join_type == JSTPJ_CROSS ||
+ plan->join_type == JSTPJ_UNION)
+ {
+ collectSiblingPathsInJsonTablePlan(plan->plan1, paths);
+ collectSiblingPathsInJsonTablePlan(plan->plan2, paths);
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE join type %d",
+ plan->join_type);
+ }
+}
+
+/*
+ * Validate child JSON_TABLE plan by checking that:
+ * - all nested columns have path names specified
+ * - all nested columns have corresponding node in the sibling plan
+ * - plan does not contain duplicate or extra nodes
+ */
+static void
+validateJsonTableChildPlan(ParseState *pstate, JsonTablePlan *plan,
+ List *columns)
+{
+ ListCell *lc1;
+ List *siblings = NIL;
+ int nchildren = 0;
+
+ if (plan)
+ collectSiblingPathsInJsonTablePlan(plan, &siblings);
+
+ foreach(lc1, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc1));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ ListCell *lc2;
+ bool found = false;
+
+ if (!jtc->pathname)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("nested JSON_TABLE columns must contain an explicit AS pathname specification if an explicit PLAN clause is used"),
+ parser_errposition(pstate, jtc->location)));
+
+ /* find nested path name in the list of sibling path names */
+ foreach(lc2, siblings)
+ {
+ if ((found = !strcmp(jtc->pathname, lfirst(lc2))))
+ break;
+ }
+
+ if (!found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node for nested path %s was not found in plan.", jtc->pathname),
+ parser_errposition(pstate, jtc->location)));
+
+ nchildren++;
+ }
+ }
+
+ if (list_length(siblings) > nchildren)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node contains some extra or duplicate sibling nodes."),
+ parser_errposition(pstate, plan ? plan->location : -1)));
+}
+
+static JsonTableColumn *
+findNestedJsonTableColumn(List *columns, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED &&
+ jtc->pathname &&
+ !strcmp(jtc->pathname, pathname))
+ return jtc;
+ }
+
+ return NULL;
+}
+
+static Node *
+transformNestedJsonTableColumn(JsonTableParseContext * cxt, JsonTableColumn *jtc,
+ JsonTablePlan *plan)
+{
+ JsonTableParent *node;
+ char *pathname = jtc->pathname;
+
+ node = transformJsonTableColumns(cxt, plan, jtc->columns, jtc->pathspec,
+ &pathname, jtc->location);
+
+ return (Node *) node;
+}
+
+static Node *
+makeJsonTableSiblingJoin(bool cross, Node *lnode, Node *rnode)
+{
+ JsonTableSibling *join = makeNode(JsonTableSibling);
+
+ join->larg = lnode;
+ join->rarg = rnode;
+ join->cross = cross;
+
+ return (Node *) join;
+}
+
+/*
+ * Recursively transform child JSON_TABLE plan.
+ *
+ * Default plan is transformed into a cross/union join of its nested columns.
+ * Simple and outer/inner plans are transformed into a JsonTableParent by
+ * finding and transforming corresponding nested column.
+ * Sibling plans are recursively transformed into a JsonTableSibling.
+ */
+static Node *
+transformJsonTableChildPlan(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns)
+{
+ JsonTableColumn *jtc = NULL;
+
+ if (!plan || plan->plan_type == JSTP_DEFAULT)
+ {
+ /* unspecified or default plan */
+ Node *res = NULL;
+ ListCell *lc;
+ bool cross = plan && (plan->join_type & JSTPJ_CROSS);
+
+ /* transform all nested columns into cross/union join */
+ foreach(lc, columns)
+ {
+ JsonTableColumn *col = castNode(JsonTableColumn, lfirst(lc));
+ Node *node;
+
+ if (col->coltype != JTC_NESTED)
+ continue;
+
+ node = transformNestedJsonTableColumn(cxt, col, plan);
+
+ /* join transformed node with previous sibling nodes */
+ res = res ? makeJsonTableSiblingJoin(cross, res, node) : node;
+ }
+
+ return res;
+ }
+ else if (plan->plan_type == JSTP_SIMPLE)
+ {
+ jtc = findNestedJsonTableColumn(columns, plan->pathname);
+ }
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ jtc = findNestedJsonTableColumn(columns, plan->plan1->pathname);
+ }
+ else
+ {
+ Node *node1 = transformJsonTableChildPlan(cxt, plan->plan1,
+ columns);
+ Node *node2 = transformJsonTableChildPlan(cxt, plan->plan2,
+ columns);
+
+ return makeJsonTableSiblingJoin(plan->join_type == JSTPJ_CROSS,
+ node1, node2);
+ }
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE plan type %d", plan->plan_type);
+
+ if (!jtc)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name was %s not found in nested columns list.",
+ plan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ return transformNestedJsonTableColumn(cxt, jtc, plan);
+}
+
+/* Check whether type is json/jsonb, array, or record. */
+static bool
+typeIsComposite(Oid typid)
+{
+ char typtype;
+
+ if (typid == JSONOID ||
+ typid == JSONBOID ||
+ typid == RECORDOID ||
+ type_is_array(typid))
+ return true;
+
+ typtype = get_typtype(typid);
+
+ if (typtype == TYPTYPE_COMPOSITE)
+ return true;
+
+ if (typtype == TYPTYPE_DOMAIN)
+ return typeIsComposite(getBaseType(typid));
+
+ return false;
+}
+
+/* Append transformed non-nested JSON_TABLE columns to the TableFunc node */
+static void
+appendJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *col;
+ ParseState *pstate = cxt->pstate;
+ JsonTable *jt = cxt->table;
+ TableFunc *tf = cxt->tablefunc;
+ bool ordinality_found = false;
+ JsonBehavior *on_error = jt->behavior ? jt->behavior->on_error : NULL;
+ bool errorOnError = on_error &&
+ on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ foreach(col, columns)
+ {
+ JsonTableColumn *rawc = castNode(JsonTableColumn, lfirst(col));
+ Oid typid;
+ int32 typmod;
+ Node *colexpr;
+
+ if (rawc->name)
+ {
+ /* make sure column names are unique */
+ ListCell *colname;
+
+ foreach(colname, tf->colnames)
+ if (!strcmp((const char *) colname, rawc->name))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("column name \"%s\" is not unique",
+ rawc->name),
+ parser_errposition(pstate, rawc->location)));
+
+ tf->colnames = lappend(tf->colnames,
+ makeString(pstrdup(rawc->name)));
+ }
+
+ /*
+ * Determine the type and typmod for the new column. FOR ORDINALITY
+ * columns are INTEGER by standard; the others are user-specified.
+ */
+ switch (rawc->coltype)
+ {
+ case JTC_FOR_ORDINALITY:
+ if (ordinality_found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use more than one FOR ORDINALITY column"),
+ parser_errposition(pstate, rawc->location)));
+ ordinality_found = true;
+ colexpr = NULL;
+ typid = INT4OID;
+ typmod = -1;
+ break;
+
+ case JTC_REGULAR:
+ typenameTypeIdAndMod(pstate, rawc->typeName, &typid, &typmod);
+
+ /*
+ * Use implicit FORMAT JSON for composite types (arrays and
+ * records).
+ *
+ * For scalar columns, require an explicit FORMAT JSON clause
+ * to get the specified WRAPPER and QUOTES behavior.
+ */
+ if (typeIsComposite(typid))
+ rawc->coltype = JTC_FORMATTED;
+ else if (rawc->wrapper != JSW_NONE)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use WITH WRAPPER clause for scalar columns without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+ else if (rawc->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use OMIT/KEEP QUOTES clause for scalar columns without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+
+ /* FALLTHROUGH */
+ case JTC_EXISTS:
+ case JTC_FORMATTED:
+ {
+ Node *je;
+ CaseTestExpr *param = makeNode(CaseTestExpr);
+
+ param->collation = InvalidOid;
+ param->typeId = cxt->contextItemTypid;
+ param->typeMod = -1;
+
+ je = transformJsonTableColumn(rawc, (Node *) param,
+ NIL, errorOnError);
+
+ colexpr = transformExpr(pstate, je, EXPR_KIND_FROM_FUNCTION);
+ assign_expr_collations(pstate, colexpr);
+
+ typid = exprType(colexpr);
+ typmod = exprTypmod(colexpr);
+ break;
+ }
+
+ case JTC_NESTED:
+ continue;
+
+ default:
+ elog(ERROR, "unknown JSON_TABLE column type: %d", rawc->coltype);
+ break;
+ }
+
+ tf->coltypes = lappend_oid(tf->coltypes, typid);
+ tf->coltypmods = lappend_int(tf->coltypmods, typmod);
+ tf->colcollations = lappend_oid(tf->colcollations, get_typcollation(typid));
+ tf->colvalexprs = lappend(tf->colvalexprs, colexpr);
+ }
+}
+
+/*
+ * Create transformed JSON_TABLE parent plan node by appending all non-nested
+ * columns to the TableFunc node and remembering their indices in the
+ * colvalexprs list.
+ */
+static JsonTableParent *
+makeParentJsonTableNode(JsonTableParseContext * cxt, char *pathSpec, char *pathName,
+ List *columns)
+{
+ JsonTableParent *node = makeNode(JsonTableParent);
+ JsonBehavior *on_error = cxt->table->behavior ?
+ cxt->table->behavior->on_error : NULL;
+
+ node->path = makeNode(JsonTablePath);
+ node->path->value = makeConst(JSONPATHOID, -1, InvalidOid, -1,
+ DirectFunctionCall1(jsonpath_in,
+ CStringGetDatum(pathSpec)),
+ false, false);
+ if (pathName)
+ node->path->name = pstrdup(pathName);
+
+ /* save start of column range */
+ node->colMin = list_length(cxt->tablefunc->colvalexprs);
+
+ appendJsonTableColumns(cxt, columns);
+
+ /* save end of column range */
+ node->colMax = list_length(cxt->tablefunc->colvalexprs) - 1;
+
+ node->errorOnError = on_error && on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ return node;
+}
+
+static JsonTableParent *
+transformJsonTableColumns(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns, char *pathSpec, char **pathName,
+ int location)
+{
+ JsonTableParent *node;
+ JsonTablePlan *childPlan;
+ bool defaultPlan = !plan || plan->plan_type == JSTP_DEFAULT;
+
+ if (!*pathName)
+ {
+ if (cxt->table->plan)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE expression"),
+ errdetail("JSON_TABLE columns must contain "
+ "explicit AS pathname specification if "
+ "explicit PLAN clause is used"),
+ parser_errposition(cxt->pstate, location)));
+
+ *pathName = generateJsonTablePathName(cxt);
+ }
+
+ if (defaultPlan)
+ childPlan = plan;
+ else
+ {
+ /* validate parent and child plans */
+ JsonTablePlan *parentPlan;
+
+ if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type != JSTPJ_INNER &&
+ plan->join_type != JSTPJ_OUTER)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Expected INNER or OUTER JSON_TABLE plan node."),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ parentPlan = plan->plan1;
+ childPlan = plan->plan2;
+
+ Assert(parentPlan->plan_type != JSTP_JOINED);
+ Assert(parentPlan->pathname);
+ }
+ else
+ {
+ parentPlan = plan;
+ childPlan = NULL;
+ }
+
+ if (strcmp(parentPlan->pathname, *pathName))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name mismatch: expected %s but %s is given.",
+ *pathName, parentPlan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ validateJsonTableChildPlan(cxt->pstate, childPlan, columns);
+ }
+
+ /* transform only non-nested columns */
+ node = makeParentJsonTableNode(cxt, pathSpec, *pathName, columns);
+
+ if (childPlan || defaultPlan)
+ {
+ /* transform recursively nested columns */
+ node->child = transformJsonTableChildPlan(cxt, childPlan, columns);
+ if (node->child)
+ node->outerJoin = !plan || (plan->join_type & JSTPJ_OUTER);
+ /* else: default plan case, no children found */
+ }
+
+ return node;
+}
+
+/*
+ * transformJsonTable -
+ * Transform a raw JsonTable into TableFunc.
+ *
+ * Transform the document-generating expression, the row-generating expression,
+ * the column-generating expressions, and the default value expressions.
+ */
+ParseNamespaceItem *
+transformJsonTable(ParseState *pstate, JsonTable *jt)
+{
+ JsonTableParseContext cxt;
+ TableFunc *tf = makeNode(TableFunc);
+ JsonFuncExpr *jfe = makeNode(JsonFuncExpr);
+ JsonExpr *je;
+ JsonTablePlan *plan = jt->plan;
+ JsonCommon *jscommon;
+ char *rootPathName = jt->common->pathname;
+ char *rootPath;
+ bool is_lateral;
+
+ cxt.pstate = pstate;
+ cxt.table = jt;
+ cxt.tablefunc = tf;
+ cxt.pathNames = NIL;
+ cxt.pathNameId = 0;
+
+ if (rootPathName)
+ registerJsonTableColumn(&cxt, rootPathName);
+
+ registerAllJsonTableColumns(&cxt, jt->columns);
+
+#if 0 /* XXX it' unclear from the standard whether
+ * root path name is mandatory or not */
+ if (plan && plan->plan_type != JSTP_DEFAULT && !rootPathName)
+ {
+ /* Assign root path name and create corresponding plan node */
+ JsonTablePlan *rootNode = makeNode(JsonTablePlan);
+ JsonTablePlan *rootPlan = (JsonTablePlan *)
+ makeJsonTableJoinedPlan(JSTPJ_OUTER, (Node *) rootNode,
+ (Node *) plan, jt->location);
+
+ rootPathName = generateJsonTablePathName(&cxt);
+
+ rootNode->plan_type = JSTP_SIMPLE;
+ rootNode->pathname = rootPathName;
+
+ plan = rootPlan;
+ }
+#endif
+
+ jscommon = copyObject(jt->common);
+ jscommon->pathspec = makeStringConst(pstrdup("$"), -1);
+
+ jfe->op = JSON_TABLE_OP;
+ jfe->common = jscommon;
+ jfe->behavior = jt->behavior;
+ jfe->location = jt->common->location;
+
+ /*
+ * We make lateral_only names of this level visible, whether or not the
+ * RangeTableFunc is explicitly marked LATERAL. This is needed for SQL
+ * spec compliance and seems useful on convenience grounds for all
+ * functions in FROM.
+ *
+ * (LATERAL can't nest within a single pstate level, so we don't need
+ * save/restore logic here.)
+ */
+ Assert(!pstate->p_lateral_active);
+ pstate->p_lateral_active = true;
+
+ tf->functype = TFT_JSON_TABLE;
+ tf->docexpr = transformExpr(pstate, (Node *) jfe, EXPR_KIND_FROM_FUNCTION);
+
+ cxt.contextItemTypid = exprType(tf->docexpr);
+
+ if (!IsA(jt->common->pathspec, A_Const) ||
+ castNode(A_Const, jt->common->pathspec)->val.node.type != T_String)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("only string constants supported in JSON_TABLE path specification"),
+ parser_errposition(pstate,
+ exprLocation(jt->common->pathspec))));
+
+ rootPath = castNode(A_Const, jt->common->pathspec)->val.sval.sval;
+
+ tf->plan = (Node *) transformJsonTableColumns(&cxt, plan, jt->columns,
+ rootPath, &rootPathName,
+ jt->common->location);
+
+ /* Also save a copy of the PASSING arguments in the TableFunc node. */
+ je = (JsonExpr *) tf->docexpr;
+ tf->passingvalexprs = copyObject(je->passing_values);
+
+ tf->ordinalitycol = -1; /* undefine ordinality column number */
+ tf->location = jt->location;
+
+ pstate->p_lateral_active = false;
+
+ /*
+ * Mark the RTE as LATERAL if the user said LATERAL explicitly, or if
+ * there are any lateral cross-references in it.
+ */
+ is_lateral = jt->lateral || contain_vars_of_level((Node *) tf, 0);
+
+ return addRangeTableEntryForTableFunc(pstate,
+ tf, jt->alias, is_lateral, true);
+}
diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c
index 864ea9b0d5..3ac7e113f0 100644
--- a/src/backend/parser/parse_relation.c
+++ b/src/backend/parser/parse_relation.c
@@ -2073,7 +2073,8 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
Assert(list_length(tf->coltypmods) == list_length(tf->colnames));
Assert(list_length(tf->colcollations) == list_length(tf->colnames));
- refname = alias ? alias->aliasname : pstrdup("xmltable");
+ refname = alias ? alias->aliasname :
+ pstrdup(tf->functype == TFT_XMLTABLE ? "xmltable" : "json_table");
rte->rtekind = RTE_TABLEFUNC;
rte->relid = InvalidOid;
@@ -2096,7 +2097,7 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
errmsg("%s function has %d columns available but %d columns specified",
- "XMLTABLE",
+ tf->functype == TFT_XMLTABLE ? "XMLTABLE" : "JSON_TABLE",
list_length(tf->colnames), numaliases)));
rte->eref = eref;
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 8f81624f97..ca7228ea41 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -2002,6 +2002,9 @@ FigureColnameInternal(Node *node, char **name)
case JSON_VALUE_OP:
*name = "json_value";
return 2;
+ case JSON_TABLE_OP:
+ *name = "json_table";
+ return 2;
}
break;
default:
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 6c5602c64d..79632e3dfd 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -61,10 +61,12 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
+#include "executor/execExpr.h"
#include "funcapi.h"
#include "lib/stringinfo.h"
#include "miscadmin.h"
#include "nodes/miscnodes.h"
+#include "nodes/nodeFuncs.h"
#include "regex/regex.h"
#include "utils/builtins.h"
#include "utils/date.h"
@@ -75,6 +77,8 @@
#include "utils/guc.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
#include "utils/timestamp.h"
#include "utils/varlena.h"
@@ -156,6 +160,61 @@ typedef struct JsonValueListIterator
ListCell *next;
} JsonValueListIterator;
+/* Structures for JSON_TABLE execution */
+
+typedef enum JsonTablePlanStateType
+{
+ JSON_TABLE_SCAN_STATE = 0,
+ JSON_TABLE_JOIN_STATE
+} JsonTablePlanStateType;
+
+typedef struct JsonTablePlanState
+{
+ JsonTablePlanStateType type;
+
+ struct JsonTablePlanState *parent;
+ struct JsonTablePlanState *nested;
+} JsonTablePlanState;
+
+typedef struct JsonTableScanState
+{
+ JsonTablePlanState plan;
+
+ MemoryContext mcxt;
+ JsonPath *path;
+ List *args;
+ JsonValueList found;
+ JsonValueListIterator iter;
+ Datum current;
+ int ordinal;
+ bool currentIsNull;
+ bool outerJoin;
+ bool errorOnError;
+ bool advanceNested;
+ bool reset;
+} JsonTableScanState;
+
+typedef struct JsonTableJoinState
+{
+ JsonTablePlanState plan;
+
+ JsonTablePlanState *left;
+ JsonTablePlanState *right;
+ bool cross;
+ bool advanceRight;
+} JsonTableJoinState;
+
+/* random number to identify JsonTableExecContext */
+#define JSON_TABLE_EXEC_CONTEXT_MAGIC 418352867
+
+typedef struct JsonTableExecContext
+{
+ int magic;
+ JsonTableScanState **colexprscans;
+ JsonTableScanState *root;
+ bool empty;
+} JsonTableExecContext;
+
/* strict/lax flags is decomposed into four [un]wrap/error flags */
#define jspStrictAbsenseOfErrors(cxt) (!(cxt)->laxMode)
#define jspAutoUnwrap(cxt) ((cxt)->laxMode)
@@ -248,6 +307,7 @@ static JsonPathExecResult getArrayIndex(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonbValue *jb, int32 *index);
static JsonBaseObjectInfo setBaseObject(JsonPathExecContext *cxt,
JsonbValue *jbv, int32 id);
+static void JsonValueListClear(JsonValueList *jvl);
static void JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv);
static int JsonValueListLength(const JsonValueList *jvl);
static bool JsonValueListIsEmpty(JsonValueList *jvl);
@@ -265,6 +325,14 @@ static JsonbValue *wrapItemsInArray(const JsonValueList *items);
static int compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
bool useTz, bool *cast_error);
+
+static JsonTablePlanState * JsonTableInitPlanState(JsonTableExecContext * cxt,
+ Node *plan,
+ JsonTablePlanState * parent);
+static bool JsonTablePlanNextRow(JsonTablePlanState * state);
+static bool JsonTableScanNextRow(JsonTableScanState *scan);
+
+
/****************** User interface to JsonPath executor ********************/
/*
@@ -2518,6 +2586,13 @@ setBaseObject(JsonPathExecContext *cxt, JsonbValue *jbv, int32 id)
return baseObject;
}
+static void
+JsonValueListClear(JsonValueList *jvl)
+{
+ jvl->singleton = NULL;
+ jvl->list = NULL;
+}
+
static void
JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv)
{
@@ -3123,3 +3198,471 @@ JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
}
}
+
+/************************ JSON_TABLE functions ***************************/
+
+/*
+ * Returns private data from executor state. Ensure validity by check with
+ * MAGIC number.
+ */
+static inline JsonTableExecContext *
+GetJsonTableExecContext(TableFuncScanState *state, const char *fname)
+{
+ JsonTableExecContext *result;
+
+ if (!IsA(state, TableFuncScanState))
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+ result = (JsonTableExecContext *) state->opaque;
+ if (result->magic != JSON_TABLE_EXEC_CONTEXT_MAGIC)
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+
+ return result;
+}
+
+/* Recursively initialize JSON_TABLE scan / join state */
+static JsonTableJoinState *
+JsonTableInitJoinState(JsonTableExecContext * cxt, JsonTableSibling *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTableJoinState *join = palloc0(sizeof(*join));
+
+ join->plan.type = JSON_TABLE_JOIN_STATE;
+ /* parent and nested not set. */
+
+ join->cross = plan->cross;
+ join->left = JsonTableInitPlanState(cxt, plan->larg, parent);
+ join->right = JsonTableInitPlanState(cxt, plan->rarg, parent);
+
+ return join;
+}
+
+static JsonTableScanState *
+JsonTableInitScanState(JsonTableExecContext * cxt, JsonTableParent *plan,
+ JsonTablePlanState * parent,
+ List *args, MemoryContext mcxt)
+{
+ JsonTableScanState *scan = palloc0(sizeof(*scan));
+ int i;
+
+ scan->plan.type = JSON_TABLE_SCAN_STATE;
+ scan->plan.parent = parent;
+
+ scan->outerJoin = plan->outerJoin;
+ scan->errorOnError = plan->errorOnError;
+ scan->path = DatumGetJsonPathP(plan->path->value->constvalue);
+ scan->args = args;
+ scan->mcxt = AllocSetContextCreate(mcxt, "JsonTableExecContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Set after settings scan->args and scan->mcxt, because the recursive
+ * call wants to use those values.
+ */
+ scan->plan.nested = plan->child ?
+ JsonTableInitPlanState(cxt, plan->child, (JsonTablePlanState *) scan) :
+ NULL;
+
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+
+ for (i = plan->colMin; i <= plan->colMax; i++)
+ cxt->colexprscans[i] = scan;
+
+ return scan;
+}
+
+/* Recursively initialize JSON_TABLE scan state */
+static JsonTablePlanState *
+JsonTableInitPlanState(JsonTableExecContext * cxt, Node *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTablePlanState *state;
+
+ if (IsA(plan, JsonTableSibling))
+ {
+ JsonTableSibling *join = castNode(JsonTableSibling, plan);
+
+ state = (JsonTablePlanState *)
+ JsonTableInitJoinState(cxt, join, parent);
+ }
+ else
+ {
+ JsonTableParent *scan = castNode(JsonTableParent, plan);
+ JsonTableScanState *parent_scan = (JsonTableScanState *) parent;
+
+ Assert(parent_scan);
+ state = (JsonTablePlanState *)
+ JsonTableInitScanState(cxt, scan, parent, parent_scan->args,
+ parent_scan->mcxt);
+ }
+
+ return state;
+}
+
+/*
+ * JsonTableInitOpaque
+ * Fill in TableFuncScanState->opaque for JsonTable processor
+ */
+static void
+JsonTableInitOpaque(TableFuncScanState *state, int natts)
+{
+ JsonTableExecContext *cxt;
+ PlanState *ps = &state->ss.ps;
+ TableFuncScan *tfs = castNode(TableFuncScan, ps->plan);
+ TableFunc *tf = tfs->tablefunc;
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+ JsonExpr *je = castNode(JsonExpr, tf->docexpr);
+ List *args = NIL;
+
+ cxt = palloc0(sizeof(JsonTableExecContext));
+ cxt->magic = JSON_TABLE_EXEC_CONTEXT_MAGIC;
+
+ if (state->passingvalexprs)
+ {
+ ListCell *exprlc;
+ ListCell *namelc;
+
+ Assert(list_length(state->passingvalexprs) ==
+ list_length(je->passing_names));
+ forboth(exprlc, state->passingvalexprs,
+ namelc, je->passing_names)
+ {
+ ExprState *state = lfirst_node(ExprState, exprlc);
+ String *name = lfirst_node(String, namelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = pstrdup(name->sval);
+ var->typid = exprType((Node *) state->expr);
+ var->typmod = exprTypmod((Node *) state->expr);
+
+ /*
+ * Evaluate the expression and save the value to be returned by
+ * GetJsonPathVar().
+ */
+ var->value = ExecEvalExpr(state, ps->ps_ExprContext,
+ &var->isnull);
+
+ args = lappend(args, var);
+ }
+ }
+
+ cxt->colexprscans = palloc(sizeof(JsonTableScanState *) *
+ list_length(tf->colvalexprs));
+
+ cxt->root = JsonTableInitScanState(cxt, root, NULL, args,
+ CurrentMemoryContext);
+ state->opaque = cxt;
+}
+
+/* Reset scan iterator to the beginning of the item list */
+static void
+JsonTableRescan(JsonTableScanState *scan)
+{
+ JsonValueListInitIterator(&scan->found, &scan->iter);
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ scan->advanceNested = false;
+ scan->ordinal = 0;
+}
+
+/* Reset context item of a scan, execute JSON path and reset a scan */
+static void
+JsonTableResetContextItem(JsonTableScanState *scan, Datum item)
+{
+ MemoryContext oldcxt;
+ JsonPathExecResult res;
+ Jsonb *js = (Jsonb *) DatumGetJsonbP(item);
+
+ JsonValueListClear(&scan->found);
+
+ MemoryContextResetOnly(scan->mcxt);
+
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+
+ res = executeJsonPath(scan->path, scan->args, GetJsonPathVar, js,
+ scan->errorOnError, &scan->found,
+ false /* FIXME */ );
+
+ MemoryContextSwitchTo(oldcxt);
+
+ if (jperIsError(res))
+ {
+ Assert(!scan->errorOnError);
+ JsonValueListClear(&scan->found); /* EMPTY ON ERROR case */
+ }
+
+ JsonTableRescan(scan);
+}
+
+/*
+ * JsonTableSetDocument
+ * Install the input document
+ */
+static void
+JsonTableSetDocument(TableFuncScanState *state, Datum value)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableSetDocument");
+
+ JsonTableResetContextItem(cxt->root, value);
+}
+
+/* Recursively reset scan and its child nodes */
+static void
+JsonTableRescanRecursive(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTableRescanRecursive(join->left);
+ JsonTableRescanRecursive(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan = (JsonTableScanState *) state;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ JsonTableRescan(scan);
+ if (scan->plan.nested)
+ JsonTableRescanRecursive(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a cross/union joined scan.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTablePlanNextRow(JsonTablePlanState * state)
+{
+ JsonTableJoinState *join;
+
+ if (state->type == JSON_TABLE_SCAN_STATE)
+ return JsonTableScanNextRow((JsonTableScanState *) state);
+
+ join = (JsonTableJoinState *) state;
+ if (join->advanceRight)
+ {
+ /* fetch next inner row */
+ if (JsonTablePlanNextRow(join->right))
+ return true;
+
+ /* inner rows are exhausted */
+ if (join->cross)
+ join->advanceRight = false; /* next outer row */
+ else
+ return false; /* end of scan */
+ }
+
+ while (!join->advanceRight)
+ {
+ /* fetch next outer row */
+ bool left = JsonTablePlanNextRow(join->left);
+
+ if (join->cross)
+ {
+ if (!left)
+ return false; /* end of scan */
+
+ JsonTableRescanRecursive(join->right);
+
+ if (!JsonTablePlanNextRow(join->right))
+ continue; /* next outer row */
+
+ join->advanceRight = true; /* next inner row */
+ }
+ else if (!left)
+ {
+ if (!JsonTablePlanNextRow(join->right))
+ return false; /* end of scan */
+
+ join->advanceRight = true; /* next inner row */
+ }
+
+ break;
+ }
+
+ return true;
+}
+
+/* Recursively set 'reset' flag of scan and its child nodes */
+static void
+JsonTablePlanReset(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTablePlanReset(join->left);
+ JsonTablePlanReset(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ scan = (JsonTableScanState *) state;
+ scan->reset = true;
+ scan->advanceNested = false;
+
+ if (scan->plan.nested)
+ JsonTablePlanReset(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a simple scan with outer/inner joined nested subscans.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTableScanNextRow(JsonTableScanState *scan)
+{
+ /* reset context item if requested */
+ if (scan->reset)
+ {
+ JsonTableScanState *parent_scan =
+ (JsonTableScanState *) scan->plan.parent;
+
+ Assert(parent_scan && !parent_scan->currentIsNull);
+ JsonTableResetContextItem(scan, parent_scan->current);
+ scan->reset = false;
+ }
+
+ if (scan->advanceNested)
+ {
+ /* fetch next nested row */
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested)
+ return true;
+ }
+
+ for (;;)
+ {
+ /* fetch next row */
+ JsonbValue *jbv = JsonValueListNext(&scan->found, &scan->iter);
+ MemoryContext oldcxt;
+
+ if (!jbv)
+ {
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ return false; /* end of scan */
+ }
+
+ /* set current row item */
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+ scan->current = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ scan->currentIsNull = false;
+ MemoryContextSwitchTo(oldcxt);
+
+ scan->ordinal++;
+
+ if (!scan->plan.nested)
+ break;
+
+ JsonTablePlanReset(scan->plan.nested);
+
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested || scan->outerJoin)
+ break;
+ }
+
+ return true;
+}
+
+/*
+ * JsonTableFetchRow
+ * Prepare the next "current" tuple for upcoming GetValue calls.
+ * Returns FALSE if the row-filter expression returned no more rows.
+ */
+static bool
+JsonTableFetchRow(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableFetchRow");
+
+ if (cxt->empty)
+ return false;
+
+ return JsonTableScanNextRow(cxt->root);
+}
+
+/*
+ * JsonTableGetValue
+ * Return the value for column number 'colnum' for the current row.
+ *
+ * This leaks memory, so be sure to reset often the context in which it's
+ * called.
+ */
+static Datum
+JsonTableGetValue(TableFuncScanState *state, int colnum,
+ Oid typid, int32 typmod, bool *isnull)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableGetValue");
+ ExprContext *econtext = state->ss.ps.ps_ExprContext;
+ ExprState *estate = list_nth(state->colvalexprs, colnum);
+ JsonTableScanState *scan = cxt->colexprscans[colnum];
+ Datum result;
+
+ if (scan->currentIsNull) /* NULL from outer/union join */
+ {
+ result = (Datum) 0;
+ *isnull = true;
+ }
+ else if (estate) /* regular column */
+ {
+ Datum saved_caseValue = econtext->caseValue_datum;
+ bool saved_caseIsNull = econtext->caseValue_isNull;
+
+ /* Pass the value for CaseTestExpr that may be present in colexpr */
+ econtext->caseValue_datum = scan->current;
+ econtext->caseValue_isNull = false;
+
+ result = ExecEvalExpr(estate, econtext, isnull);
+
+ econtext->caseValue_datum = saved_caseValue;
+ econtext->caseValue_isNull = saved_caseIsNull;
+ }
+ else
+ {
+ result = Int32GetDatum(scan->ordinal); /* ordinality column */
+ *isnull = false;
+ }
+
+ return result;
+}
+
+/*
+ * JsonTableDestroyOpaque
+ */
+static void
+JsonTableDestroyOpaque(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableDestroyOpaque");
+
+ /* not valid anymore */
+ cxt->magic = 0;
+
+ state->opaque = NULL;
+}
+
+const TableFuncRoutine JsonbTableRoutine =
+{
+ JsonTableInitOpaque,
+ JsonTableSetDocument,
+ NULL,
+ NULL,
+ NULL,
+ JsonTableFetchRow,
+ JsonTableGetValue,
+ JsonTableDestroyOpaque
+};
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 342a2cf6aa..ca13107536 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -522,6 +522,8 @@ static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
static void get_json_path_spec(Node *path_spec, deparse_context *context,
bool showimplicit);
+static void get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8627,7 +8629,8 @@ get_json_behavior(JsonBehavior *behavior, deparse_context *context,
/*
* get_json_expr_options
*
- * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS and
+ * JSON_TABLE columns.
*/
static void
get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
@@ -9875,6 +9878,9 @@ get_rule_expr(Node *node, deparse_context *context,
case JSON_VALUE_OP:
appendStringInfoString(buf, "JSON_VALUE(");
break;
+ default:
+ elog(ERROR, "unexpected JsonExpr type: %d", jexpr->op);
+ break;
}
get_rule_expr(jexpr->formatted_expr, context, showimplicit);
@@ -11241,16 +11247,14 @@ get_sublink_expr(SubLink *sublink, deparse_context *context)
/* ----------
- * get_tablefunc - Parse back a table function
+ * get_xmltable - Parse back a XMLTABLE function
* ----------
*/
static void
-get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+get_xmltable(TableFunc *tf, deparse_context *context, bool showimplicit)
{
StringInfo buf = context->buf;
- /* XMLTABLE is the only existing implementation. */
-
appendStringInfoString(buf, "XMLTABLE(");
if (tf->ns_uris != NIL)
@@ -11341,6 +11345,271 @@ get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
appendStringInfoChar(buf, ')');
}
+/*
+ * get_json_nested_columns - Parse back nested JSON_TABLE columns
+ */
+static void
+get_json_table_nested_columns(TableFunc *tf, Node *node,
+ deparse_context *context, bool showimplicit,
+ bool needcomma)
+{
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_nested_columns(tf, n->larg, context, showimplicit,
+ needcomma);
+ get_json_table_nested_columns(tf, n->rarg, context, showimplicit, true);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ if (needcomma)
+ appendStringInfoChar(context->buf, ',');
+
+ appendStringInfoChar(context->buf, ' ');
+ appendContextKeyword(context, "NESTED PATH ", 0, 0, 0);
+ get_const_expr(n->path->value, context, -1);
+ appendStringInfo(context->buf, " AS %s", quote_identifier(n->path->name));
+ get_json_table_columns(tf, n, context, showimplicit);
+ }
+}
+
+/*
+ * get_json_table_plan - Parse back a JSON_TABLE plan
+ */
+static void
+get_json_table_plan(TableFunc *tf, Node *node, deparse_context *context,
+ bool parenthesize)
+{
+ if (parenthesize)
+ appendStringInfoChar(context->buf, '(');
+
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_plan(tf, n->larg, context,
+ IsA(n->larg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->larg)->child);
+
+ appendStringInfoString(context->buf, n->cross ? " CROSS " : " UNION ");
+
+ get_json_table_plan(tf, n->rarg, context,
+ IsA(n->rarg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->rarg)->child);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ appendStringInfoString(context->buf, quote_identifier(n->path->name));
+
+ if (n->child)
+ {
+ appendStringInfoString(context->buf,
+ n->outerJoin ? " OUTER " : " INNER ");
+ get_json_table_plan(tf, n->child, context,
+ IsA(n->child, JsonTableSibling));
+ }
+ }
+
+ if (parenthesize)
+ appendStringInfoChar(context->buf, ')');
+}
+
+/*
+ * get_json_table_columns - Parse back JSON_TABLE columns
+ */
+static void
+get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ ListCell *lc_colname;
+ ListCell *lc_coltype;
+ ListCell *lc_coltypmod;
+ ListCell *lc_colvalexpr;
+ int colnum = 0;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "COLUMNS (", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forfour(lc_colname, tf->colnames,
+ lc_coltype, tf->coltypes,
+ lc_coltypmod, tf->coltypmods,
+ lc_colvalexpr, tf->colvalexprs)
+ {
+ char *colname = strVal(lfirst(lc_colname));
+ JsonExpr *colexpr;
+ Oid typid;
+ int32 typmod;
+ bool ordinality;
+ JsonBehaviorType default_behavior;
+
+ typid = lfirst_oid(lc_coltype);
+ typmod = lfirst_int(lc_coltypmod);
+ colexpr = castNode(JsonExpr, lfirst(lc_colvalexpr));
+
+ if (colnum < node->colMin)
+ {
+ colnum++;
+ continue;
+ }
+
+ if (colnum > node->colMax)
+ break;
+
+ if (colnum > node->colMin)
+ appendStringInfoString(buf, ", ");
+
+ colnum++;
+
+ ordinality = !colexpr;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ appendStringInfo(buf, "%s %s", quote_identifier(colname),
+ ordinality ? "FOR ORDINALITY" :
+ format_type_with_typemod(typid, typmod));
+ if (ordinality)
+ continue;
+
+ if (colexpr->op == JSON_EXISTS_OP)
+ {
+ appendStringInfoString(buf, " EXISTS");
+ default_behavior = JSON_BEHAVIOR_FALSE;
+ }
+ else
+ {
+ if (colexpr->op == JSON_QUERY_OP)
+ {
+ char typcategory;
+ bool typispreferred;
+
+ get_type_category_preferred(typid, &typcategory, &typispreferred);
+
+ if (typcategory == TYPCATEGORY_STRING)
+ appendStringInfoString(buf,
+ colexpr->format->format_type == JS_FORMAT_JSONB ?
+ " FORMAT JSONB" : " FORMAT JSON");
+ }
+
+ default_behavior = JSON_BEHAVIOR_NULL;
+ }
+
+ if (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR)
+ default_behavior = JSON_BEHAVIOR_ERROR;
+
+ appendStringInfoString(buf, " PATH ");
+
+ get_json_path_spec(colexpr->path_spec, context, showimplicit);
+
+ get_json_expr_options(colexpr, context, default_behavior);
+ }
+
+ if (node->child)
+ get_json_table_nested_columns(tf, node->child, context, showimplicit,
+ node->colMax >= node->colMin);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_json_table - Parse back a JSON_TABLE function
+ * ----------
+ */
+static void
+get_json_table(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+
+ appendStringInfoString(buf, "JSON_TABLE(");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_const_expr(root->path->value, context, -1);
+
+ appendStringInfo(buf, " AS %s", quote_identifier(root->path->name));
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PASSING ", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr((Node *) lfirst(lc2), context, false);
+ appendStringInfo(buf, " AS %s",
+ quote_identifier((lfirst_node(String, lc1))->sval)
+ );
+ }
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+ }
+
+ get_json_table_columns(tf, root, context, showimplicit);
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PLAN ", 0, 0, 0);
+ get_json_table_plan(tf, (Node *) root, context, true);
+
+ if (jexpr->on_error->btype != JSON_BEHAVIOR_EMPTY)
+ get_json_behavior(jexpr->on_error, context, "ERROR");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_tablefunc - Parse back a table function
+ * ----------
+ */
+static void
+get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ /* XMLTABLE and JSON_TABLE are the only existing implementations. */
+
+ if (tf->functype == TFT_XMLTABLE)
+ get_xmltable(tf, context, showimplicit);
+ else if (tf->functype == TFT_JSON_TABLE)
+ get_json_table(tf, context, showimplicit);
+}
+
/* ----------
* get_from_clause - Parse back a FROM clause
*
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 6a7118d300..2fa0328977 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1882,6 +1882,8 @@ typedef struct TableFuncScanState
ExprState *rowexpr; /* state for row-generating expression */
List *colexprs; /* state for column-generating expression */
List *coldefexprs; /* state for column default expressions */
+ List *colvalexprs; /* state for column value expression */
+ List *passingvalexprs; /* state for PASSING argument expression */
List *ns_names; /* same as TableFunc.ns_names */
List *ns_uris; /* list of states of namespace URI exprs */
Bitmapset *notnulls; /* nullability flag for each output column */
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 783e82ac92..79c486eceb 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -116,6 +116,8 @@ extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int loc
extern JsonBehaviorClause *makeJsonBehaviorClause(JsonBehavior *on_empty,
JsonBehavior *on_error,
int location);
+extern Node *makeJsonTableJoinedPlan(JsonTablePlanJoinType type,
+ Node *plan1, Node *plan2, int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ce6893fb91..7ed60a7d2c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1721,6 +1721,19 @@ typedef struct JsonBehaviorClause
*/
typedef char *JsonPathSpec;
+/*
+ * JsonTableColumnType -
+ * enumeration of JSON_TABLE column types
+ */
+typedef enum JsonTableColumnType
+{
+ JTC_FOR_ORDINALITY,
+ JTC_REGULAR,
+ JTC_EXISTS,
+ JTC_FORMATTED,
+ JTC_NESTED,
+} JsonTableColumnType;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1773,6 +1786,82 @@ typedef struct JsonFuncExpr
int location; /* token location, or -1 if unknown */
} JsonFuncExpr;
+/*
+ * JsonTableColumn -
+ * untransformed representation of JSON_TABLE column
+ */
+typedef struct JsonTableColumn
+{
+ NodeTag type;
+ JsonTableColumnType coltype; /* column type */
+ char *name; /* column name */
+ TypeName *typeName; /* column type name */
+ char *pathspec; /* path specification, if any */
+ char *pathname; /* path name, if any */
+ JsonFormat *format; /* JSON format clause, if specified */
+ JsonWrapper wrapper; /* WRAPPER behavior for formatted columns */
+ JsonQuotes quotes; /* omit or keep quotes on scalar strings? */
+ List *columns; /* nested columns */
+ JsonBehaviorClause *behavior; /* ON ERROR / EMPTY behavior, if specified */
+ int location; /* token location, or -1 if unknown */
+} JsonTableColumn;
+
+/*
+ * JsonTablePlanType -
+ * flags for JSON_TABLE plan node types representation
+ */
+typedef enum JsonTablePlanType
+{
+ JSTP_DEFAULT,
+ JSTP_SIMPLE,
+ JSTP_JOINED,
+} JsonTablePlanType;
+
+/*
+ * JsonTablePlanJoinType -
+ * flags for JSON_TABLE join types representation
+ */
+typedef enum JsonTablePlanJoinType
+{
+ JSTPJ_INNER = 0x01,
+ JSTPJ_OUTER = 0x02,
+ JSTPJ_CROSS = 0x04,
+ JSTPJ_UNION = 0x08,
+} JsonTablePlanJoinType;
+
+typedef struct JsonTablePlan JsonTablePlan;
+
+/*
+ * JsonTablePlan -
+ * untransformed representation of JSON_TABLE plan node
+ */
+struct JsonTablePlan
+{
+ NodeTag type;
+ JsonTablePlanType plan_type; /* plan type */
+ JsonTablePlanJoinType join_type; /* join type (for joined plan only) */
+ JsonTablePlan *plan1; /* first joined plan */
+ JsonTablePlan *plan2; /* second joined plan */
+ char *pathname; /* path name (for simple plan only) */
+ int location; /* token location, or -1 if unknown */
+};
+
+/*
+ * JsonTable -
+ * untransformed representation of JSON_TABLE
+ */
+typedef struct JsonTable
+{
+ NodeTag type;
+ JsonCommon *common; /* common JSON path syntax fields */
+ List *columns; /* list of JsonTableColumn */
+ JsonTablePlan *plan; /* join plan, if specified */
+ JsonBehaviorClause *behavior; /* ON ERROR behavior, if specified */
+ Alias *alias; /* table alias in FROM clause */
+ bool lateral; /* does it have LATERAL prefix? */
+ int location; /* token location, or -1 if unknown */
+} JsonTable;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 3f75c4608e..4fe392bddb 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -94,8 +94,14 @@ typedef struct RangeVar
int location;
} RangeVar;
+typedef enum TableFuncType
+{
+ TFT_XMLTABLE,
+ TFT_JSON_TABLE
+} TableFuncType;
+
/*
- * TableFunc - node for a table function, such as XMLTABLE.
+ * TableFunc - node for a table function, such as XMLTABLE or JSON_TABLE.
*
* Entries in the ns_names list are either String nodes containing
* literal namespace names, or NULL pointers to represent DEFAULT.
@@ -103,6 +109,8 @@ typedef struct RangeVar
typedef struct TableFunc
{
NodeTag type;
+ /* XMLTABLE or JSON_TABLE */
+ TableFuncType functype;
/* list of namespace URI expressions */
List *ns_uris pg_node_attr(query_jumble_ignore);
/* list of namespace names or NULL */
@@ -123,8 +131,14 @@ typedef struct TableFunc
List *colexprs;
/* list of column default expressions */
List *coldefexprs pg_node_attr(query_jumble_ignore);
+ /* list of column value expressions */
+ List *colvalexprs pg_node_attr(query_jumble_ignore);
+ /* list of PASSING argument expressions */
+ List *passingvalexprs pg_node_attr(query_jumble_ignore);
/* nullability flag for each output column */
Bitmapset *notnulls pg_node_attr(query_jumble_ignore);
+ /* JSON_TABLE plan */
+ Node *plan pg_node_attr(query_jumble_ignore);
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1560,7 +1574,8 @@ typedef enum JsonExprOp
{
JSON_VALUE_OP, /* JSON_VALUE() */
JSON_QUERY_OP, /* JSON_QUERY() */
- JSON_EXISTS_OP /* JSON_EXISTS() */
+ JSON_EXISTS_OP, /* JSON_EXISTS() */
+ JSON_TABLE_OP /* JSON_TABLE() */
} JsonExprOp;
/*
@@ -1795,6 +1810,48 @@ typedef struct JsonExpr
int location; /* token location, or -1 if unknown */
} JsonExpr;
+/*
+ * JsonTablePath
+ * A JSON path expression to be computed as part of evaluating
+ * a JSON_TABLE plan node
+ */
+typedef struct JsonTablePath
+{
+ NodeTag type;
+
+ Const *value;
+ char *name;
+} JsonTablePath;
+
+/*
+ * JsonTableParent -
+ * transformed representation of parent JSON_TABLE plan node
+ */
+typedef struct JsonTableParent
+{
+ NodeTag type;
+ JsonTablePath *path;
+ Node *child; /* nested columns, if any */
+ bool outerJoin; /* outer or inner join for nested columns? */
+ int colMin; /* min column index in the resulting column
+ * list */
+ int colMax; /* max column index in the resulting column
+ * list */
+ bool errorOnError; /* ERROR/EMPTY ON ERROR behavior */
+} JsonTableParent;
+
+/*
+ * JsonTableSibling -
+ * transformed representation of joined sibling JSON_TABLE plan node
+ */
+typedef struct JsonTableSibling
+{
+ NodeTag type;
+ Node *larg; /* left join node */
+ Node *rarg; /* right join node */
+ bool cross; /* cross or union join? */
+} JsonTableSibling;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 0954d9fc7b..f88a6c9ac6 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -242,6 +242,7 @@ PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_table", JSON_TABLE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -284,6 +285,7 @@ PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("nested", NESTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -334,7 +336,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_clause.h b/src/include/parser/parse_clause.h
index 8e26482db4..7152a8ef48 100644
--- a/src/include/parser/parse_clause.h
+++ b/src/include/parser/parse_clause.h
@@ -51,4 +51,7 @@ extern List *addTargetToSortList(ParseState *pstate, TargetEntry *tle,
extern Index assignSortGroupRef(TargetEntry *tle, List *tlist);
extern bool targetIsInSortList(TargetEntry *tle, Oid sortop, List *sortList);
+/* functions in parse_jsontable.c */
+extern ParseNamespaceItem *transformJsonTable(ParseState *pstate, JsonTable *jt);
+
#endif /* PARSE_CLAUSE_H */
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index 5a37133847..7eb0ccf4e4 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -15,6 +15,7 @@
#define JSONPATH_H
#include "fmgr.h"
+#include "executor/tablefunc.h"
#include "nodes/pg_list.h"
#include "nodes/primnodes.h"
#include "utils/jsonb.h"
@@ -288,4 +289,6 @@ extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
bool *error, List *vars);
+extern PGDLLIMPORT const TableFuncRoutine JsonbTableRoutine;
+
#endif
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
index 8b87580752..995f267404 100644
--- a/src/test/regress/expected/json_sqljson.out
+++ b/src/test/regress/expected/json_sqljson.out
@@ -16,3 +16,9 @@ ERROR: JSON_QUERY() is not yet implemented for the json type
LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
^
HINT: Try casting the argument to jsonb
+-- JSON_TABLE
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
+ERROR: JSON_TABLE() is not yet implemented for the json type
+LINE 1: SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo ...
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
index 9dc71cfb38..1a5f437ff9 100644
--- a/src/test/regress/expected/jsonb_sqljson.out
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -1093,3 +1093,1189 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.dateti
ERROR: functions in index expression must be marked IMMUTABLE
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+-- JSON_TABLE
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+ERROR: syntax error at or near "("
+LINE 1: SELECT JSON_TABLE('[]', '$');
+ ^
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ERROR: syntax error at or near ")"
+LINE 1: SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ ^
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+ERROR: JSON_TABLE function has 1 columns available but 2 columns specified
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+ foo
+-----
+(0 rows)
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+ item | foo
+------+-----
+ 123 |
+(1 row)
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | int | text | char(4) | bool | numeric | domain | js | jb
+---------------------------------------------------------------------------------------+----+-----+---------+---------+------+---------+---------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [] | | | | | | | | |
+ {} | 1 | | | | | | | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1 | 1.23 | 1.23 | | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | 2 | 2 | 2 | | 2 | 2 | "2" | "2"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | aaaaaaa | aaaa | | | aaaaaaa | "aaaaaaa" | "aaaaaaa"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | foo | foo | | | | "foo" | "foo"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | | | | | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | 0 | false | fals | f | | false | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | 1 | true | true | t | | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | | | | | | | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | [1,2] | [1,2 | | | [1,2] | "[1,2]" | "[1,2]"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | "str" | "str | | | "str" | "\"str\"" | "\"str\""
+(14 rows)
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+ js | id | jst | jsc | jsv | jsb | jsbq
+---------------------------------------------------------------------------------------+----+--------------+------+------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | 1 | 1
+ [] | | | | | |
+ {} | 1 | {} | {} | {} | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1.23 | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | "2" | "2" | "2" | "2" | 2
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | "aaaaaaa" | "aaa | "aaa | "aaaaaaa" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | "foo" | "foo | "foo | "foo" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | null | null | null | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | false | fals | fals | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | true | true | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | {"aaa": 123} | {"aa | {"aa | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | "[1,2]" | "[1, | "[1, | "[1,2]" | [1, 2]
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | "\"str\"" | "\"s | "\"s | "\"str\"" | "str"
+(14 rows)
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+ js | id | exists1 | exists2 | exists3 | exists4
+---------------------------------------------------------------------------------------+----+---------+---------+---------+---------
+ 1 | 1 | f | 0 | | false
+ [] | | | | |
+ {} | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | t | 1 | 1 | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | f | 0 | | false
+(14 rows)
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | aaa | aaa1 | js2 | jsb2w | jsb2q | ia | ta | jba
+---------------------------------------------------------------------------------------+----+-----+------+--------------+----------------+--------------+----+----+-----
+ 1 | 1 | | | 1 | [1] | 1 | | |
+ [] | | | | | | | | |
+ {} | 1 | | | {} | [{}] | {} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | | | 1 | [1] | 1 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | | | 1.23 | [1.23] | 1.23 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | | | "2" | ["2"] | 2 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | | "aaaaaaa" | ["aaaaaaa"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | | "foo" | ["foo"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | null | [null] | null | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | | | false | [false] | false | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | | | true | [true] | true | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | 123 | 123 | {"aaa": 123} | [{"aaa": 123}] | {"aaa": 123} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | | "[1,2]" | ["[1,2]"] | [1, 2] | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | | "\"str\"" | ["\"str\""] | "str" | | |
+(14 rows)
+
+-- JSON_TABLE: Test backward parsing
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+\sv jsonb_table_view
+CREATE OR REPLACE VIEW public.jsonb_table_view AS
+ SELECT id,
+ "int",
+ text,
+ "char(4)",
+ bool,
+ "numeric",
+ domain,
+ js,
+ jb,
+ jst,
+ jsc,
+ jsv,
+ jsb,
+ jsbq,
+ aaa,
+ aaa1,
+ exists1,
+ exists2,
+ exists3,
+ js2,
+ jsb2w,
+ jsb2q,
+ ia,
+ ta,
+ jba,
+ a1,
+ b1,
+ a11,
+ a21,
+ a22
+ FROM JSON_TABLE(
+ 'null'::jsonb, '$[*]' AS json_table_path_1
+ PASSING
+ 1 + 2 AS a,
+ '"foo"'::json AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" integer PATH '$',
+ text text PATH '$',
+ "char(4)" character(4) PATH '$',
+ bool boolean PATH '$',
+ "numeric" numeric PATH '$',
+ domain jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc character(4) FORMAT JSON PATH '$',
+ jsv character varying(4) FORMAT JSON PATH '$',
+ jsb jsonb PATH '$',
+ jsbq jsonb PATH '$' OMIT QUOTES,
+ aaa integer PATH '$."aaa"',
+ aaa1 integer PATH '$."aaa"',
+ exists1 boolean EXISTS PATH '$."aaa"',
+ exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia integer[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1
+ COLUMNS (
+ a1 integer PATH '$."a1"',
+ b1 text PATH '$."b1"',
+ NESTED PATH '$[*]' AS "p1 1"
+ COLUMNS (
+ a11 text PATH '$."a11"'
+ )
+ ),
+ NESTED PATH '$[2]' AS p2
+ COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1"
+ COLUMNS (
+ a21 text PATH '$."a21"'
+ ),
+ NESTED PATH '$[*]' AS p22
+ COLUMNS (
+ a22 text PATH '$."a22"'
+ )
+ )
+ )
+ PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22))))
+ )
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+ QUERY PLAN
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ Table Function Scan on "json_table"
+ Output: "json_table".id, "json_table"."int", "json_table".text, "json_table"."char(4)", "json_table".bool, "json_table"."numeric", "json_table".domain, "json_table".js, "json_table".jb, "json_table".jst, "json_table".jsc, "json_table".jsv, "json_table".jsb, "json_table".jsbq, "json_table".aaa, "json_table".aaa1, "json_table".exists1, "json_table".exists2, "json_table".exists3, "json_table".js2, "json_table".jsb2w, "json_table".jsb2q, "json_table".ia, "json_table".ta, "json_table".jba, "json_table".a1, "json_table".b1, "json_table".a11, "json_table".a21, "json_table".a22
+ Table Function Call: JSON_TABLE('null'::jsonb, '$[*]' AS json_table_path_1 PASSING 3 AS a, '"foo"'::jsonb AS "b c" COLUMNS (id FOR ORDINALITY, "int" integer PATH '$', text text PATH '$', "char(4)" character(4) PATH '$', bool boolean PATH '$', "numeric" numeric PATH '$', domain jsonb_test_domain PATH '$', js json PATH '$', jb jsonb PATH '$', jst text FORMAT JSON PATH '$', jsc character(4) FORMAT JSON PATH '$', jsv character varying(4) FORMAT JSON PATH '$', jsb jsonb PATH '$', jsbq jsonb PATH '$' OMIT QUOTES, aaa integer PATH '$."aaa"', aaa1 integer PATH '$."aaa"', exists1 boolean EXISTS PATH '$."aaa"', exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR, exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR, js2 json PATH '$', jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER, jsb2q jsonb PATH '$' OMIT QUOTES, ia integer[] PATH '$', ta text[] PATH '$', jba jsonb[] PATH '$', NESTED PATH '$[1]' AS p1 COLUMNS (a1 integer PATH '$."a1"', b1 text PATH '$."b1"', NESTED PATH '$[*]' AS "p1 1" COLUMNS (a11 text PATH '$."a11"')), NESTED PATH '$[2]' AS p2 COLUMNS ( NESTED PATH '$[*]' AS "p2:1" COLUMNS (a21 text PATH '$."a21"'), NESTED PATH '$[*]' AS p22 COLUMNS (a22 text PATH '$."a22"'))) PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22)))))
+(3 rows)
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+ERROR: cannot use more than one FOR ORDINALITY column
+LINE 1: ..._TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR OR...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+ id | a
+----+---
+ 1 | 1
+(1 row)
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+ js | a
+-------+---
+ 1 | 1
+ "err" |
+(2 rows)
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+ a
+---
+
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: no SQL/JSON item
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 1
+(1 row)
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+ a
+---
+ 0
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to smallint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to bigint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to real
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 E...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+ a
+-----
+ fal
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to json
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to jsonb
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EX...
+ ^
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+ERROR: cannot use OMIT/KEEP QUOTES clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+ item
+---------
+ "world"
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+ERROR: cannot use WITH WRAPPER clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+ item
+-----------
+ ["world"]
+(1 row)
+
+-- JSON_TABLE: nested paths and plans
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb '[]', '$' -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 4: NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: b
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+-- JSON_TABLE: plan validation
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p1)
+ ^
+DETAIL: Path name mismatch: expected p0 but p1 is given.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 UNION p1 UNION p11)
+ ^
+DETAIL: Expected INNER or OUTER JSON_TABLE plan node.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 8: NESTED PATH '$' AS p2 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p2 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 5: NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ ^
+DETAIL: Plan node for nested path p11 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+ ^
+DETAIL: Plan node contains some extra or duplicate sibling nodes.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 6: NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ^
+DETAIL: Plan node for nested path p12 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 9: NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ ^
+DETAIL: Plan node for nested path p21 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+ bar | foo | baz
+-----+-----+-----
+(0 rows)
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb 'null', 'strict $[*]' -- without root path name
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- JSON_TABLE: plan execution
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+ n | a | c | b
+---+----+----+---
+ 1 | 1 | |
+ 2 | 2 | 10 |
+ 2 | 2 | |
+ 2 | 2 | 20 |
+ 2 | 2 | | 1
+ 2 | 2 | | 2
+ 2 | 2 | | 3
+ 3 | 3 | | 1
+ 3 | 3 | | 2
+ 4 | -1 | | 1
+ 4 | -1 | | 2
+(11 rows)
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+ n | a | b | b1 | c | c1 | b
+---+---+--------------+-----+------+----+-----
+ 1 | 1 | [1, 10] | 1 | 1 | | 101
+ 1 | 1 | [1, 10] | 1 | null | | 101
+ 1 | 1 | [1, 10] | 1 | 2 | | 101
+ 1 | 1 | [1, 10] | 10 | 1 | | 110
+ 1 | 1 | [1, 10] | 10 | null | | 110
+ 1 | 1 | [1, 10] | 10 | 2 | | 110
+ 1 | 1 | [2] | 2 | 1 | | 102
+ 1 | 1 | [2] | 2 | null | | 102
+ 1 | 1 | [2] | 2 | 2 | | 102
+ 1 | 1 | [3, 30, 300] | 3 | 1 | | 103
+ 1 | 1 | [3, 30, 300] | 3 | null | | 103
+ 1 | 1 | [3, 30, 300] | 3 | 2 | | 103
+ 1 | 1 | [3, 30, 300] | 30 | 1 | | 130
+ 1 | 1 | [3, 30, 300] | 30 | null | | 130
+ 1 | 1 | [3, 30, 300] | 30 | 2 | | 130
+ 1 | 1 | [3, 30, 300] | 300 | 1 | | 400
+ 1 | 1 | [3, 30, 300] | 300 | null | | 400
+ 1 | 1 | [3, 30, 300] | 300 | 2 | | 400
+ 2 | 2 | | | | |
+ 3 | | | | | |
+(20 rows)
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+ x | y | y | z
+---+---+--------------+---
+ 2 | 1 | [1, 2, 3] | 1
+ 2 | 1 | [1, 2, 3] | 2
+ 2 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [1, 2, 3] | 1
+ 3 | 1 | [1, 2, 3] | 2
+ 3 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [2, 3, 4, 5] | 2
+ 3 | 1 | [2, 3, 4, 5] | 3
+ 3 | 1 | [2, 3, 4, 5] | 4
+ 3 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [1, 2, 3] | 1
+ 4 | 1 | [1, 2, 3] | 2
+ 4 | 1 | [1, 2, 3] | 3
+ 4 | 1 | [2, 3, 4, 5] | 2
+ 4 | 1 | [2, 3, 4, 5] | 3
+ 4 | 1 | [2, 3, 4, 5] | 4
+ 4 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [3, 4, 5, 6] | 3
+ 4 | 1 | [3, 4, 5, 6] | 4
+ 4 | 1 | [3, 4, 5, 6] | 5
+ 4 | 1 | [3, 4, 5, 6] | 6
+ 2 | 2 | [1, 2, 3] | 2
+ 2 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [1, 2, 3] | 2
+ 3 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [2, 3, 4, 5] | 2
+ 3 | 2 | [2, 3, 4, 5] | 3
+ 3 | 2 | [2, 3, 4, 5] | 4
+ 3 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [1, 2, 3] | 2
+ 4 | 2 | [1, 2, 3] | 3
+ 4 | 2 | [2, 3, 4, 5] | 2
+ 4 | 2 | [2, 3, 4, 5] | 3
+ 4 | 2 | [2, 3, 4, 5] | 4
+ 4 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [3, 4, 5, 6] | 3
+ 4 | 2 | [3, 4, 5, 6] | 4
+ 4 | 2 | [3, 4, 5, 6] | 5
+ 4 | 2 | [3, 4, 5, 6] | 6
+ 2 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [2, 3, 4, 5] | 3
+ 3 | 3 | [2, 3, 4, 5] | 4
+ 3 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [1, 2, 3] | 3
+ 4 | 3 | [2, 3, 4, 5] | 3
+ 4 | 3 | [2, 3, 4, 5] | 4
+ 4 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [3, 4, 5, 6] | 3
+ 4 | 3 | [3, 4, 5, 6] | 4
+ 4 | 3 | [3, 4, 5, 6] | 5
+ 4 | 3 | [3, 4, 5, 6] | 6
+(52 rows)
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+ERROR: could not find jsonpath variable "x"
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+ json_value
+------------
+ foo
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_query
+------------
+ 123
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+ json_query
+------------
+ [123]
+(1 row)
+
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+ERROR: syntax error at or near " " of jsonpath input
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
+ERROR: only string constants supported in JSON_TABLE path specification
+LINE 1: SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || '...
+ ^
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
index 4f30fa46b9..df4a430d88 100644
--- a/src/test/regress/sql/json_sqljson.sql
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -9,3 +9,7 @@ SELECT JSON_VALUE(NULL FORMAT JSON, '$');
-- JSON_QUERY
SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+
+-- JSON_TABLE
+
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
index 5c5d69a74e..4001545c29 100644
--- a/src/test/regress/sql/jsonb_sqljson.sql
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -346,3 +346,677 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime()
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+
+-- JSON_TABLE
+
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- JSON_TABLE: Test backward parsing
+
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+
+\sv jsonb_table_view
+
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+
+-- JSON_TABLE: nested paths and plans
+
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+
+-- JSON_TABLE: plan validation
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+
+-- JSON_TABLE: plan execution
+
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 5ce706205c..e4e525f940 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1311,6 +1311,7 @@ JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
+JsonPathSpec
JsonPathString
JsonPathVarCallback
JsonPathVariable
@@ -1320,6 +1321,17 @@ JsonReturning
JsonScalarExpr
JsonSemAction
JsonSerializeExpr
+JsonTable
+JsonTableColumn
+JsonTableColumnType
+JsonTableContext
+JsonTableJoinState
+JsonTableParent
+JsonTablePlan
+JsonTablePlanJoinType
+JsonTablePlanType
+JsonTableScanState
+JsonTableSibling
JsonTokenType
JsonTransformStringValuesAction
JsonTypeCategory
@@ -2783,6 +2795,7 @@ TableFunc
TableFuncRoutine
TableFuncScan
TableFuncScanState
+TableFuncType
TableInfo
TableLikeClause
TableSampleClause
--
2.35.3
[application/octet-stream] v24-0003-SQL-JSON-query-functions.patch (199.2K, ../../CA+HiwqG_XFksg7HN_gbmcOQU00Q5bvTKeLSZ3NftewdBRw4fKA@mail.gmail.com/6-v24-0003-SQL-JSON-query-functions.patch)
download | inline diff:
From 43f3b4a4f99f136bf77d1e5d914bcc0c2b2bf979 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:12 +0900
Subject: [PATCH v24 3/5] SQL/JSON query functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This introduces the SQL/JSON functions for querying JSON data using
jsonpath expressions. The functions are:
JSON_EXISTS()
JSON_QUERY()
JSON_VALUE()
JSON_EXISTS() tests if the jsonpath expression applied to the jsonb
value yields any values.
JSON_VALUE() must return a single value, and an error occurs if it
tries to return multiple values.
JSON_QUERY() must return a json object or array, and there are
various WRAPPER options for handling scalar or multi-value results.
Both these functions have options for handling EMPTY and ERROR
conditions.
All of these functions only operate on jsonb. The workaround for now
is to cast the argument to jsonb.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he, Anton A. Melnikov, Nikita Malakhov
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 152 +++
src/backend/executor/execExpr.c | 500 +++++++++
src/backend/executor/execExprInterp.c | 399 +++++++
src/backend/jit/llvm/llvmjit.c | 2 +
src/backend/jit/llvm/llvmjit_expr.c | 124 +++
src/backend/jit/llvm/llvmjit_types.c | 4 +
src/backend/nodes/makefuncs.c | 33 +
src/backend/nodes/nodeFuncs.c | 179 +++
src/backend/optimizer/path/costsize.c | 3 +-
src/backend/optimizer/util/clauses.c | 19 +
src/backend/parser/gram.y | 217 +++-
src/backend/parser/parse_expr.c | 553 +++++++++-
src/backend/parser/parse_target.c | 15 +
src/backend/utils/adt/formatting.c | 44 +
src/backend/utils/adt/jsonb.c | 62 ++
src/backend/utils/adt/jsonfuncs.c | 52 +-
src/backend/utils/adt/jsonpath.c | 255 +++++
src/backend/utils/adt/jsonpath_exec.c | 391 ++++++-
src/backend/utils/adt/ruleutils.c | 137 +++
src/include/executor/execExpr.h | 113 ++
src/include/fmgr.h | 1 +
src/include/jit/llvmjit.h | 1 +
src/include/nodes/makefuncs.h | 4 +
src/include/nodes/parsenodes.h | 70 ++
src/include/nodes/primnodes.h | 125 +++
src/include/parser/kwlist.h | 11 +
src/include/utils/formatting.h | 1 +
src/include/utils/jsonb.h | 3 +
src/include/utils/jsonfuncs.h | 5 +
src/include/utils/jsonpath.h | 27 +
src/interfaces/ecpg/preproc/ecpg.trailer | 28 +
src/test/regress/expected/json_sqljson.out | 18 +
src/test/regress/expected/jsonb_sqljson.out | 1095 +++++++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/json_sqljson.sql | 11 +
src/test/regress/sql/jsonb_sqljson.sql | 348 ++++++
src/tools/pgindent/typedefs.list | 20 +
37 files changed, 4957 insertions(+), 67 deletions(-)
create mode 100644 src/test/regress/expected/json_sqljson.out
create mode 100644 src/test/regress/expected/jsonb_sqljson.out
create mode 100644 src/test/regress/sql/json_sqljson.sql
create mode 100644 src/test/regress/sql/jsonb_sqljson.sql
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 202e64d0e0..c12ee06713 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -18170,6 +18170,158 @@ $.* ? (@ like_regex "^\\d+$")
</programlisting>
</para>
</sect3>
+
+ <sect3 id="sqljson-query-functions">
+ <title>SQL/JSON Query Functions</title>
+ <para>
+ <xref linkend="functions-sqljson-querying"/> details the SQL/JSON
+ functions that can be used to query JSON data.
+ </para>
+
+ <note>
+ <para>
+ SQL/JSON path expression can currently only accept values of the
+ <type>jsonb</type> type, so it might be necessary to cast the
+ <replaceable>context_item</replaceable> argument of these functions to
+ <type>jsonb</type>.
+ </para>
+ </note>
+
+ <table id="functions-sqljson-querying">
+ <title>SQL/JSON Query Functions</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ Function signature
+ </para>
+ <para>
+ Description
+ </para>
+ <para>
+ Example(s)
+ </para></entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_exists</primary></indexterm>
+ <function>json_exists</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> </optional>
+ <optional> { <literal>TRUE</literal> | <literal>FALSE</literal> |<literal> UNKNOWN</literal> | <literal>ERROR</literal> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns true if the SQL/JSON <replaceable>path_expression</replaceable>
+ applied to the <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s yields any items.
+ The <literal>ON ERROR</literal> clause specifies what is returned if
+ an error occurs; the default is to return <literal>FALSE</literal>.
+ Note that if the <replaceable>path_expression</replaceable>
+ is <literal>strict</literal>, an error is generated if it yields no items.
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"key1": [1,2,3]}', 'strict $.key1[*] ? (@ > 2)')</literal>
+ <returnvalue>t</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'lax $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>f</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'strict $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>ERROR: jsonpath array subscript is out of bounds</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_query</primary></indexterm>
+ <function>json_query</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> <optional> <literal>FORMAT JSON</literal> <optional> <literal>ENCODING UTF8</literal> </optional> </optional> </optional>
+ <optional> { <literal>WITHOUT</literal> | <literal>WITH</literal> { <literal>CONDITIONAL</literal> | <optional><literal>UNCONDITIONAL</literal></optional> } } <optional> <literal>ARRAY</literal> </optional> <literal>WRAPPER</literal> </optional>
+ <optional> { <literal>KEEP</literal> | <literal>OMIT</literal> } <literal>QUOTES</literal> <optional> <literal>ON SCALAR STRING</literal> </optional> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s.
+ This function must return a JSON string, so if the path expression
+ returns multiple SQL/JSON items, you must wrap the result using the
+ <literal>WITH WRAPPER</literal> clause. If the wrapper is
+ <literal>UNCONDITIONAL</literal>, an array wrapper will always
+ be applied, even if the returned value is already a single JSON object
+ or an array, but if it is <literal>CONDITIONAL</literal>, it will not be
+ applied to a single array or object. <literal>UNCONDITIONAL</literal>
+ is the default. If the result is a scalar string, by default the value
+ returned will have surrounding quotes making it a valid JSON value,
+ which can be made explicit by specifying <literal>KEEP QUOTES</literal>.
+ Conversely, quotes can be omitted by specifying <literal>OMIT QUOTES</literal>.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>;
+ the default returned type is <type>jsonb</type>.
+ The <literal>ON EMPTY</literal> clause specifies the behavior if the
+ <replaceable>path_expression</replaceable> yields no value at all; the
+ default when <literal>ON EMPTY</literal> is not specified is to return a
+ null value. The <literal>ON ERROR</literal> clause specifies the behavior
+ if an error occurs as a result of <type>jsonpath</type> evaluation
+ (including cast to the output type) or during the execution of
+ <literal>ON EMPTY</literal> behavior (that was caused by empty result of
+ <type>jsonpath</type> evaluation); the default when
+ <literal>ON ERROR</literal> is not specified is to return a null value.
+ </para>
+ <para>
+ <literal>json_query(jsonb '[1,[2,3],null]', 'lax $[*][1]' WITH CONDITIONAL WRAPPER)</literal>
+ <returnvalue>[3]</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_value</primary></indexterm>
+ <function>json_value</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable>
+ <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <literal>PASSING</literal> <replaceable>value</replaceable>s. The
+ extracted value must be a single <acronym>SQL/JSON</acronym> scalar
+ item. For results that are objects or arrays, use the
+ <function>json_query</function> function instead.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>.
+ The default returned type is <type>text</type>.
+ The <literal>ON ERROR</literal> and <literal>ON EMPTY</literal>
+ clauses have similar semantics as mentioned in the description of
+ <function>json_query</function>.
+ </para>
+ <para>
+ <literal>json_value(jsonb '"123.45"', '$' RETURNING float)</literal>
+ <returnvalue>123.45</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '"03:04 2015-02-01"', '$.datetime("HH24:MI YYYY-MM-DD")' RETURNING date)</literal>
+ <returnvalue>2015-02-01</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '[1,2]', 'strict $[*]' DEFAULT 9 ON ERROR)</literal>
+ <returnvalue>9</returnvalue>
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect3>
</sect2>
</sect1>
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 34bd2102b5..cfbd92720d 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -49,6 +49,7 @@
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/typcache.h"
@@ -88,6 +89,15 @@ static void ExecBuildAggTransCall(ExprState *state, AggState *aggstate,
FunctionCallInfo fcinfo, AggStatePerTrans pertrans,
int transno, int setno, int setoff, bool ishash,
bool nullcheck);
+static void ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch);
+static Datum GetJsonBehaviorConstVal(JsonBehavior *behavior, bool *is_null);
+static int ExecInitJsonCoercion(ExprEvalStep *scratch, ExprState *state,
+ JsonExprState *jsestate,
+ JsonCoercion *coercion,
+ bool throw_errors,
+ Datum *resv, bool *resnull);
/*
@@ -2416,6 +2426,14 @@ ExecInitExprRec(Expr *node, ExprState *state,
break;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+
+ ExecInitJsonExpr(jexpr, state, resv, resnull, &scratch);
+ break;
+ }
+
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -4184,3 +4202,485 @@ ExecBuildParamSetEqual(TupleDesc desc,
return state;
}
+
+/*
+ * Push steps to evaluate a JsonExpr and its various subsidiary expressions.
+ */
+static void
+ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch)
+{
+ JsonExprState *jsestate = palloc0(sizeof(JsonExprState));
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ ListCell *argexprlc;
+ ListCell *argnamelc;
+ int coercion_step_off = -1;
+ int on_error_step_off = -1;
+ List *jumps_if_skip = NIL;
+ List *jumps_to_coerce_finish = NIL;
+ List *jumps_to_coerce_or_end = NIL;
+ List *jumps_to_end = NIL;
+ ListCell *lc;
+ ExprEvalStep *as;
+
+ jsestate->jsexpr = jexpr;
+
+ /*
+ * Evaluate formatted_expr, storing the result into
+ * jsestate->formatted_expr.
+ */
+ ExecInitExprRec((Expr *) jexpr->formatted_expr, state,
+ &jsestate->formatted_expr.value,
+ &jsestate->formatted_expr.isnull);
+
+ /*
+ * Steps to JUMP to end if formatted_expr evaluated to NULL, skipping over
+ * next steps including the JsonPath evaluation.
+ */
+ jumps_if_skip = lappend_int(jumps_if_skip, state->steps_len);
+ scratch->opcode = EEOP_JUMP_IF_NULL;
+ scratch->resnull = &jsestate->formatted_expr.isnull;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Evaluate pathspec expression, storing the result into
+ * jsestate->pathspec.
+ */
+ ExecInitExprRec((Expr *) jexpr->path_spec, state,
+ &jsestate->pathspec.value,
+ &jsestate->pathspec.isnull);
+
+ /*
+ * Steps to JUMP to end if pathspec evaluated to NULL, skipping over
+ * next steps including the JsonPath evaluation.
+ */
+ jumps_if_skip = lappend_int(jumps_if_skip, state->steps_len);
+ scratch->opcode = EEOP_JUMP_IF_NULL;
+ scratch->resnull = &jsestate->pathspec.isnull;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+
+ /* Steps to compute PASSING args. */
+ jsestate->args = NIL;
+ forboth(argexprlc, jexpr->passing_values,
+ argnamelc, jexpr->passing_names)
+ {
+ Expr *argexpr = (Expr *) lfirst(argexprlc);
+ String *argname = lfirst_node(String, argnamelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = argname->sval;
+ var->typid = exprType((Node *) argexpr);
+ var->typmod = exprTypmod((Node *) argexpr);
+
+ ExecInitExprRec((Expr *) argexpr, state, &var->value, &var->isnull);
+
+ jsestate->args = lappend(jsestate->args, var);
+ }
+
+ /*
+ * Step for the actual JsonPath* evaluation; see ExecEvalJsonExprPath().
+ */
+ scratch->opcode = EEOP_JSONEXPR_PATH;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Step to jump to end when there's neither an error nor any need to coerce
+ * the JsonPath* result.
+ */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Steps to coerce the JsonPath* result computed by
+ * ExecEvalJsonExprPath(). To handle coercion errors softly, use the
+ * following ErrorSaveContext when initializing the coercion expressions
+ * and in ExecEvalJsonCoercionViaPopulateOrIO().
+ */
+ jsestate->escontext.type = T_ErrorSaveContext;
+ if (jexpr->result_coercion || jexpr->omit_quotes)
+ {
+ coercion_step_off = state->steps_len;
+
+ jsestate->jump_eval_result_coercion =
+ ExecInitJsonCoercion(scratch, state, jsestate,
+ jexpr->result_coercion,
+ jexpr->on_error->btype == JSON_BEHAVIOR_ERROR,
+ resv, resnull);
+
+ /*
+ * Step to jump to the EEOP_JSONEXPR_FINISH step skipping over item
+ * coercion steps that will be added below, if any.
+ */
+ if (jexpr->item_coercions)
+ {
+ jumps_to_coerce_finish = lappend_int(jumps_to_coerce_finish,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+ else
+ jsestate->jump_eval_result_coercion = -1;
+
+ /* Steps for coercions of JsonItemType values for JSON_VALUE_OP. */
+ if (jexpr->item_coercions)
+ {
+ if (coercion_step_off < 0)
+ coercion_step_off = state->steps_len;
+
+ /*
+ * ExecPrepareJsonItemCoercion() called by ExecEvalJsonExprPath()
+ * chooses one from the array for a given JsonbValue returned by
+ * JsonPathValue(), indexed by JsonItemType of a given
+ * JsonbValue.
+ */
+ jsestate->num_item_coercions = list_length(jexpr->item_coercions);
+ jsestate->eval_item_coercion_jumps = (int *)
+ palloc(jsestate->num_item_coercions * sizeof(int));
+ jsestate->item_coercion_via_expr = (bool *)
+ palloc(jsestate->num_item_coercions * sizeof(bool));
+ foreach(lc, jexpr->item_coercions)
+ {
+ JsonItemCoercion *item_coercion = lfirst(lc);
+ JsonCoercion *coercion = item_coercion->coercion;
+
+ jsestate->item_coercion_via_expr[item_coercion->item_type] =
+ item_coercion->coercion->expr != NULL;
+ jsestate->eval_item_coercion_jumps[item_coercion->item_type] =
+ ExecInitJsonCoercion(scratch, state, jsestate,
+ coercion,
+ jexpr->on_error->btype == JSON_BEHAVIOR_ERROR,
+ resv, resnull);
+
+ /* Emit JUMP step to skip past other coercions' steps. */
+ jumps_to_coerce_finish = lappend_int(jumps_to_coerce_finish,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /*
+ * And a step to clean up after the coercion step; see
+ * ExecEvalJsonCoercionFinish(). Its main role is to check if an
+ * error occurred when evaluating the coercion and handle it per the
+ * specified ON ERROR behavior.
+ */
+ if (coercion_step_off >= 0)
+ {
+ foreach(lc, jumps_to_coerce_finish)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+
+ scratch->opcode = EEOP_JSONEXPR_COERCION_FINISH;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+ }
+
+ /*
+ * Step to handle non-ERROR ON ERROR behaviors. That also handles errors
+ * that may occur during coercion handling.
+ */
+ if (jexpr->on_error &&
+ jexpr->on_error->btype != JSON_BEHAVIOR_ERROR)
+ {
+ on_error_step_off = state->steps_len;
+ if (jexpr->on_empty == NULL ||
+ jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
+ jumps_to_end = lappend_int(jumps_to_end, on_error_step_off);
+ scratch->opcode = EEOP_JUMP_IF_NOT_TRUE;
+ /*
+ * post_eval.error is set as appropriate by ExecEvalJsonExprPath() and
+ * ExecEvalJsonCoercionFinish() to trigger the ON ERROR step.
+ */
+ scratch->resvalue = &post_eval->error.value;
+ scratch->resnull = &post_eval->error.isnull;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /* Step(s) to evaluate the ON ERROR expression */
+ if (jexpr->on_error->default_expr)
+ {
+ ErrorSaveContext *save_escontext = state->escontext;
+
+ state->escontext = &jsestate->escontext;
+ ExecInitExprRec((Expr *) jexpr->on_error->default_expr,
+ state, resv, resnull);
+ state->escontext = save_escontext;
+
+ /* Jump to end, because no coercion needed. */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ ExprEvalPushStep(state, scratch);
+ }
+ else
+ {
+ Datum constvalue;
+ bool constisnull;
+
+ constvalue = GetJsonBehaviorConstVal(jexpr->on_error,
+ &constisnull);
+ scratch->opcode = EEOP_CONST;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.constval.value = constvalue;
+ scratch->d.constval.isnull = constisnull;
+
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Jump to the coercion step to coerce the above value to the
+ * desired output type.
+ */
+ jumps_to_coerce_or_end = lappend_int(jumps_to_coerce_or_end,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /* Step to handle non-ERROR ON EMPTY behaviors. */
+ if (jexpr->on_empty != NULL &&
+ jexpr->on_empty->btype != JSON_BEHAVIOR_ERROR)
+ {
+ /* Jump to handle ON EMPTY after checking error. */
+ if (on_error_step_off >= 0)
+ {
+ as = &state->steps[on_error_step_off];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+ /*
+ * If no step was added above, jump to handle ON EMPTY directly
+ * instead.
+ */
+ else
+ on_error_step_off = state->steps_len;
+
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP_IF_NOT_TRUE;
+ scratch->resvalue = &post_eval->empty.value;
+ scratch->resnull = &post_eval->empty.isnull;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /* Step(s) to evaluate the ON EMPTY expression */
+ if (jexpr->on_empty->default_expr)
+ {
+ ErrorSaveContext *save_escontext = state->escontext;
+
+ state->escontext = &jsestate->escontext;
+ ExecInitExprRec((Expr *) jexpr->on_empty->default_expr,
+ state, resv, resnull);
+ state->escontext = save_escontext;
+
+ /*
+ * Emit JUMP step to jump to the end to skip over the CONST step
+ * that will be added below. Note no coercion needed.
+ */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+ }
+ else
+ {
+ Datum constvalue;
+ bool constisnull;
+
+ constvalue = GetJsonBehaviorConstVal(jexpr->on_empty,
+ &constisnull);
+ scratch->opcode = EEOP_CONST;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.constval.value = constvalue;
+ scratch->d.constval.isnull = constisnull;
+
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Emit JUMP step to jump to the coercion step to coerce the above
+ * value to the desired output type.
+ */
+ jumps_to_coerce_or_end = lappend_int(jumps_to_coerce_or_end,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /*
+ * Return NULL on skipping JsonPath* evaluation when either formatted_expr
+ * or pathspec is NULL.
+ */
+ foreach(lc, jumps_if_skip)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+ scratch->opcode = EEOP_CONST;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.constval.value = (Datum) 0;
+ scratch->d.constval.isnull = true;
+ ExprEvalPushStep(state, scratch);
+
+ /* Do put that NULL through coercion though. */
+ jumps_to_coerce_or_end = lappend_int(jumps_to_coerce_or_end,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Adjust remaining jump target addresses now that we have the necessary
+ * steps in place.
+ */
+ Assert(on_error_step_off >= 0 || jexpr == NULL ||
+ jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
+ jsestate->jump_error = on_error_step_off;
+
+ /* Adjust EEOP_JUMP steps */
+ foreach(lc, jumps_to_coerce_or_end)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = coercion_step_off >= 0 ?
+ coercion_step_off : state->steps_len;
+ }
+
+ foreach(lc, jumps_to_end)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+
+ /*
+ * Set information for RETURNING type's input function used by
+ * ExecEvalJsonExprCoercion().
+ */
+ if (jexpr->omit_quotes ||
+ (jexpr->result_coercion && jexpr->result_coercion->via_io))
+ {
+ Oid typinput;
+ FmgrInfo *finfo;
+
+ /* lookup the result type's input function */
+ getTypeInputInfo(jexpr->returning->typid, &typinput,
+ &jsestate->input.typioparam);
+ finfo = palloc0(sizeof(FmgrInfo));
+ fmgr_info(typinput, finfo);
+ jsestate->input.finfo = finfo;
+ }
+}
+
+/*
+ * Returns constant values to be returned to the user for various
+ * non-ERROR ON ERROR/EMPTY behaviors.
+ *
+ * Note that JSON_BEHAVIOR_DEFAULT expression is handled by the
+ * caller separately.
+ */
+static Datum
+GetJsonBehaviorConstVal(JsonBehavior *behavior, bool *is_null)
+{
+ *is_null = false;
+
+ switch (behavior->btype)
+ {
+ case JSON_BEHAVIOR_EMPTY_ARRAY:
+ return JsonbPGetDatum(JsonbMakeEmptyArray());
+
+ case JSON_BEHAVIOR_EMPTY_OBJECT:
+ return JsonbPGetDatum(JsonbMakeEmptyObject());
+
+ case JSON_BEHAVIOR_TRUE:
+ return BoolGetDatum(true);
+
+ case JSON_BEHAVIOR_FALSE:
+ return BoolGetDatum(false);
+
+ case JSON_BEHAVIOR_NULL:
+ case JSON_BEHAVIOR_UNKNOWN:
+ case JSON_BEHAVIOR_EMPTY:
+ *is_null = true;
+ return (Datum) 0;
+
+ case JSON_BEHAVIOR_DEFAULT:
+ /* Always handled in the caller. */
+ Assert(false);
+ return (Datum) 0;
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON behavior %d", behavior->btype);
+ return (Datum) 0;
+ }
+}
+
+/* Initialize one JsonCoercion for execution. */
+static int
+ExecInitJsonCoercion(ExprEvalStep *scratch, ExprState *state,
+ JsonExprState *jsestate,
+ JsonCoercion *coercion,
+ bool throw_errors,
+ Datum *resv, bool *resnull)
+{
+ int jump_eval_coercion;
+
+ if (jsestate->jsexpr->omit_quotes ||
+ (coercion && (coercion->via_io || coercion->via_populate)))
+ {
+ jump_eval_coercion = state->steps_len;
+ scratch->opcode = EEOP_JSONEXPR_COERCION;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+ }
+ else if (coercion && coercion->expr)
+ {
+ Datum *save_innermost_caseval;
+ bool *save_innermost_casenull;
+ ErrorSaveContext *save_escontext;
+
+ /* Push step(s) to compute cstate->coercion. */
+ save_innermost_caseval = state->innermost_caseval;
+ save_innermost_casenull = state->innermost_casenull;
+ save_escontext = state->escontext;
+
+ state->innermost_caseval = resv;
+ state->innermost_casenull = resnull;
+ if (!throw_errors)
+ state->escontext = &jsestate->escontext;
+ else
+ state->escontext = NULL;
+
+ jump_eval_coercion = state->steps_len;
+ ExecInitExprRec((Expr *) coercion->expr, state, resv, resnull);
+
+ state->innermost_caseval = save_innermost_caseval;
+ state->innermost_casenull = save_innermost_casenull;
+ state->escontext = save_escontext;
+ }
+ else
+ {
+ /*
+ * Coercion is unnecessary; for example, RETURNING type matches JSON
+ * item's type exactly.
+ */
+ jump_eval_coercion = -1;
+ }
+
+ return jump_eval_coercion;
+}
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 4e152fdfe3..92e15fdc7f 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -75,6 +75,7 @@
#include "utils/json.h"
#include "utils/jsonb.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/timestamp.h"
@@ -153,6 +154,10 @@ static TupleDesc get_cached_rowtype(Oid type_id, int32 typmod,
bool *changed);
static void ExecEvalRowNullInt(ExprState *state, ExprEvalStep *op,
ExprContext *econtext, bool checkisnull);
+static Datum ExecPrepareJsonItemCoercion(JsonbValue *item,
+ JsonExprState *jsestate,
+ int *jumps_eval_item_coercion,
+ bool *via_expr);
/* fast-path evaluation functions */
static Datum ExecJustInnerVar(ExprState *state, ExprContext *econtext, bool *isnull);
@@ -482,6 +487,9 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_XMLEXPR,
&&CASE_EEOP_JSON_CONSTRUCTOR,
&&CASE_EEOP_IS_JSON,
+ &&CASE_EEOP_JSONEXPR_PATH,
+ &&CASE_EEOP_JSONEXPR_COERCION,
+ &&CASE_EEOP_JSONEXPR_COERCION_FINISH,
&&CASE_EEOP_AGGREF,
&&CASE_EEOP_GROUPING_FUNC,
&&CASE_EEOP_WINDOW_FUNC,
@@ -1551,6 +1559,35 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_JSONEXPR_PATH)
+ {
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+
+ /* too complex for an inline implementation */
+ if (!ExecEvalJsonExprPath(state, op, econtext))
+ EEO_JUMP(jsestate->jump_error);
+ else if (jsestate->post_eval.jump_eval_coercion >= 0)
+ EEO_JUMP(jsestate->post_eval.jump_eval_coercion);
+
+ EEO_NEXT();
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION)
+ {
+ /* too complex for an inline implementation */
+ ExecEvalJsonCoercionViaPopulateOrIO(state, op, econtext);
+
+ EEO_NEXT();
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION_FINISH)
+ {
+ /* too complex for an inline implementation */
+ ExecEvalJsonCoercionFinish(state, op);
+
+ EEO_NEXT();
+ }
+
EEO_CASE(EEOP_AGGREF)
{
/*
@@ -4208,6 +4245,368 @@ ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op)
*op->resvalue = BoolGetDatum(res);
}
+/*
+ * Performs JsonPath{Exists|Query|Value}() for given context item and JSON
+ * path.
+ *
+ * Result is set in *op->resvalue and *op->resnull.
+ *
+ * On return, JsonExprPostEvalState is populated with the following details:
+ * - jump_eval_coercion: step address of coercion to apply to the result
+ * - error.value: true if an error occurred during JsonPath evaluation
+ * - empty.value: true if JsonPath{Query|Value}() found no matching item
+ *
+ * No return if the ON ERROR/EMPTY behavior is ERROR.
+ */
+bool
+ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonExpr *jexpr = jsestate->jsexpr;
+ Datum item;
+ JsonPath *path;
+ bool throw_error = (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
+ bool error = false,
+ empty = false;
+
+ item = jsestate->formatted_expr.value;
+ path = DatumGetJsonPathP(jsestate->pathspec.value);
+
+ /* Reset JsonExprPostEvalState for this evaluation. */
+ memset(post_eval, 0, sizeof(*post_eval));
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ {
+ bool exists = JsonPathExists(item, path,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (!error)
+ {
+ *op->resvalue = BoolGetDatum(exists);
+ *op->resnull = false;
+ }
+ }
+ break;
+
+ case JSON_QUERY_OP:
+ *op->resvalue = JsonPathQuery(item, path, jexpr->wrapper, &empty,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (!error && !empty)
+ *op->resnull = (DatumGetPointer(*op->resvalue) == NULL);
+ break;
+
+ case JSON_VALUE_OP:
+ {
+ JsonbValue *jbv = JsonPathValue(item, path, &empty,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ /* Might get overridden by an item coercion below. */
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (jbv == NULL)
+ {
+ /* Will be coerced with result_coercion. */
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ }
+ else if (!error && !empty)
+ {
+ Assert(jbv != NULL);
+
+ /*
+ * If the requested output type is json(b), use
+ * result_coercion to do the coercion.
+ */
+ if (jexpr->returning->typid == JSONOID ||
+ jexpr->returning->typid == JSONBOID)
+ {
+ *op->resvalue = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ *op->resnull = false;
+ }
+ else
+ {
+ bool via_expr;
+ int jump_eval_item_coercion;
+
+ /*
+ * Else, use one of the item_coercions.
+ *
+ * Error out if no cast expression exists.
+ */
+ *op->resvalue = ExecPrepareJsonItemCoercion(jbv, jsestate,
+ &jump_eval_item_coercion,
+ &via_expr);
+ *op->resnull = false;
+ if (jump_eval_item_coercion >= 0 && !via_expr)
+ {
+ if (!throw_error)
+ {
+ /* Will be coerced with result_coercion. */
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ }
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_ITEM_CANNOT_BE_CAST_TO_TARGET_TYPE),
+ errmsg("SQL/JSON item cannot be cast to target type")));
+ }
+ post_eval->jump_eval_coercion = jump_eval_item_coercion;
+ }
+ }
+ break;
+ }
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
+ return false;
+ }
+
+ if (error)
+ {
+ Assert(!throw_error && jsestate->jump_error >= 0);
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ post_eval->error.value = BoolGetDatum(true);
+ return false;
+ }
+
+ if (empty)
+ {
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ if (jexpr->on_empty &&
+ jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
+ {
+ if (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR)
+ ereport(ERROR,
+ (errcode(ERRCODE_NO_SQL_JSON_ITEM),
+ errmsg("no SQL/JSON item")));
+ post_eval->error.value = BoolGetDatum(true);
+ }
+
+ post_eval->empty.value = BoolGetDatum(true);
+ return false;
+ }
+
+ post_eval->coercing_jsonpath_item = true;
+ return true;
+}
+
+
+/*
+ * Evaluate or return the step address to evaluate a coercion of a JSON item
+ * to the target type. The former if the coercion must be done right away by
+ * calling the target type's input function, and for some types, by calling
+ * json_populate_type().
+ *
+ * Returns the step address to be performed next.
+ */
+void
+ExecEvalJsonCoercionViaPopulateOrIO(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonExpr *jexpr = jsestate->jsexpr;
+ Node *escontext_p = jexpr->on_error->btype != JSON_BEHAVIOR_ERROR ?
+ (Node *) &jsestate->escontext : NULL;
+ JsonCoercion *coercion = jexpr->result_coercion;
+ Datum res = *op->resvalue;
+ bool resnull = *op->resnull;
+ bool type_is_domain =
+ (getBaseType(jexpr->returning->typid) != jexpr->returning->typid);
+
+ Assert(coercion != NULL || jexpr->omit_quotes);
+
+ /*
+ * Force-throw an error if the returning type is a domain because its
+ * constraint violations must be reported, or if coercing a behavior
+ * value.
+ */
+ if (type_is_domain || !post_eval->coercing_jsonpath_item)
+ escontext_p = NULL;
+
+ if (coercion && coercion->via_populate)
+ {
+ *op->resvalue = json_populate_type(res, JSONBOID,
+ jexpr->returning->typid,
+ jexpr->returning->typmod,
+ &jsestate->cache,
+ econtext->ecxt_per_query_memory,
+ op->resnull,
+ escontext_p);
+ if (SOFT_ERROR_OCCURRED(escontext_p))
+ {
+ post_eval->error.value = BoolGetDatum(true);
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ }
+ }
+ else if ((coercion && coercion->via_io) || jexpr->omit_quotes)
+ {
+ char *val_string = resnull ? NULL :
+ JsonbUnquote(DatumGetJsonbP(res));
+
+ if (!InputFunctionCallSafe(jsestate->input.finfo, val_string,
+ jsestate->input.typioparam,
+ jexpr->returning->typmod,
+ escontext_p,
+ op->resvalue))
+ {
+ post_eval->error.value = BoolGetDatum(true);
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ }
+ }
+}
+
+/*
+ * Checks if the coercion evaluation (either an expression or
+ * ExecEvalJsonCoercionViaPopulateOrIO()) led to an error. If an error did
+ * occur, this sets post_eval->error to trigger the subsequent ON ERROR
+ * handling steps.
+ */
+void
+ExecEvalJsonCoercionFinish(ExprState *state, ExprEvalStep *op)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+
+ if (SOFT_ERROR_OCCURRED(&jsestate->escontext))
+ {
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ jsestate->post_eval.error.value = BoolGetDatum(true);
+
+ /*
+ * Also make ErrorSaveContext ready for the next row. Since we never
+ * set details_wanted, we don't need to also reset error_data, which
+ * would be NULL anyway.
+ */
+ Assert(!jsestate->escontext.details_wanted &&
+ jsestate->escontext.error_data == NULL);
+ jsestate->escontext.error_occurred = false;
+ }
+ else
+ {
+ /*
+ * If no error, reset error/empty flags so that ON ERROR/EMPTY handling
+ * steps are skipped over.
+ */
+ jsestate->post_eval.error.value = BoolGetDatum(false);
+ jsestate->post_eval.empty.value = BoolGetDatum(false);
+ }
+}
+
+/*
+ * Prepare SQL/JSON item coercion to the output type. Returned a datum of the
+ * corresponding SQL type and a pointer to the coercion state.
+ */
+static Datum
+ExecPrepareJsonItemCoercion(JsonbValue *item, JsonExprState *jsestate,
+ int *jump_eval_item_coercion,
+ bool *via_expr)
+{
+ int *eval_item_coercion_jumps = jsestate->eval_item_coercion_jumps;
+ bool *item_coercion_via_expr = jsestate->item_coercion_via_expr;
+ int jump_to;
+ Datum res;
+ JsonbValue buf;
+
+ if (item->type == jbvBinary &&
+ JsonContainerIsScalar(item->val.binary.data))
+ {
+ bool res PG_USED_FOR_ASSERTS_ONLY;
+
+ res = JsonbExtractScalar(item->val.binary.data, &buf);
+ item = &buf;
+ Assert(res);
+ }
+
+ /* get coercion state reference and datum of the corresponding SQL type */
+ switch (item->type)
+ {
+ case jbvNull:
+ *via_expr = item_coercion_via_expr[JsonItemTypeNull];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeNull];
+ res = (Datum) 0;
+ break;
+
+ case jbvString:
+ *via_expr = item_coercion_via_expr[JsonItemTypeString];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeString];
+ res = PointerGetDatum(cstring_to_text_with_len(item->val.string.val,
+ item->val.string.len));
+ break;
+
+ case jbvNumeric:
+ *via_expr = item_coercion_via_expr[JsonItemTypeNumeric];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeNumeric];
+ res = NumericGetDatum(item->val.numeric);
+ break;
+
+ case jbvBool:
+ *via_expr = item_coercion_via_expr[JsonItemTypeBoolean];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeBoolean];
+ res = BoolGetDatum(item->val.boolean);
+ break;
+
+ case jbvDatetime:
+ res = item->val.datetime.value;
+ switch (item->val.datetime.typid)
+ {
+ case DATEOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeDate];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeDate];
+ break;
+ case TIMEOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTime];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTime];
+ break;
+ case TIMETZOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTimetz];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimetz];
+ break;
+ case TIMESTAMPOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTimestamp];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimestamp];
+ break;
+ case TIMESTAMPTZOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTimestamptz];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimestamptz];
+ break;
+ default:
+ elog(ERROR, "unexpected jsonb datetime type oid %u",
+ item->val.datetime.typid);
+ return (Datum) 0;
+ }
+ break;
+
+ case jbvArray:
+ case jbvObject:
+ case jbvBinary:
+ *via_expr = item_coercion_via_expr[JsonItemTypeComposite];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeComposite];
+ res = JsonbPGetDatum(JsonbValueToJsonb(item));
+ break;
+
+ default:
+ elog(ERROR, "unexpected jsonb value type %d", item->type);
+ return (Datum) 0;
+ }
+
+ *jump_eval_item_coercion = jump_to;
+
+ return res;
+}
+
/*
* ExecEvalGroupingFunc
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 2c8ac02550..4e83e5ef7f 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -84,6 +84,7 @@ LLVMTypeRef StructAggState;
LLVMTypeRef StructAggStatePerGroupData;
LLVMTypeRef StructAggStatePerTransData;
LLVMTypeRef StructPlanState;
+LLVMTypeRef StructJsonExprPostEvalState;
LLVMValueRef AttributeTemplate;
LLVMValueRef ExecEvalSubroutineTemplate;
@@ -1186,6 +1187,7 @@ llvm_create_types(void)
StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
StructPlanState = llvm_pg_var_type("StructPlanState");
StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
+ StructJsonExprPostEvalState = llvm_pg_var_type("StructJsonExprPostEvalState");
AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index 81856a9dc7..901324c988 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1930,6 +1930,130 @@ llvm_compile_expr(ExprState *state)
LLVMBuildBr(b, opblocks[opno + 1]);
break;
+ case EEOP_JSONEXPR_PATH:
+ {
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ LLVMValueRef v_ret;
+ LLVMBasicBlockRef b_coercion;
+
+ b_coercion =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_coercion", opno);
+
+ v_ret = build_EvalXFunc(b, mod, "ExecEvalJsonExprPath",
+ v_state, op, v_econtext);
+ v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, "");
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_sbool_const(0),
+ ""),
+ jsestate->jump_error >= 0 ?
+ opblocks[jsestate->jump_error] :
+ b_coercion,
+ b_coercion);
+
+ LLVMPositionBuilderAtEnd(b, b_coercion);
+ if (jsestate->jump_eval_result_coercion >= 0 ||
+ jsestate->num_item_coercions > 0)
+ {
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ int i;
+ LLVMValueRef v_post_eval;
+ LLVMValueRef v_post_eval_jump_eval_coercionp;
+ LLVMValueRef v_post_eval_jump_eval_coercion;
+ LLVMValueRef v_jump_coercion;
+ LLVMValueRef v_switch;
+ LLVMBasicBlockRef b_done,
+ b_result_coercion_block,
+ *b_item_coercion_blocks = NULL;
+
+ v_post_eval = l_ptr_const(post_eval, l_ptr(StructJsonExprPostEvalState));
+ v_post_eval_jump_eval_coercionp =
+ l_struct_gep(b,
+ StructJsonExprPostEvalState,
+ v_post_eval,
+ FIELDNO_JSONEXPRPOSTEVALSTATE_JUMP_EVAL_COERCION,
+ "v_post_eval_jump_eval_coercion");
+ v_post_eval_jump_eval_coercion =
+ l_load(b, LLVMInt32TypeInContext(lc),
+ v_post_eval_jump_eval_coercionp, "");
+
+ b_result_coercion_block =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_result_coercion", opno);
+ if (jsestate->num_item_coercions > 0)
+ {
+ b_item_coercion_blocks = palloc(sizeof(LLVMBasicBlockRef) *
+ jsestate->num_item_coercions);
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ b_item_coercion_blocks[i] =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_item_coercion.%d",
+ opno, i);
+ }
+ }
+ b_done =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_done", opno);
+
+ v_switch = LLVMBuildSwitch(b,
+ v_post_eval_jump_eval_coercion,
+ b_done,
+ jsestate->num_item_coercions + 1);
+ if (jsestate->jump_eval_result_coercion >= 0)
+ {
+ v_jump_coercion = l_int32_const(lc, jsestate->jump_eval_result_coercion);
+ LLVMAddCase(v_switch, v_jump_coercion, b_result_coercion_block);
+ }
+
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ if (jsestate->eval_item_coercion_jumps[i] >= 0)
+ {
+ v_jump_coercion = l_int32_const(lc, jsestate->eval_item_coercion_jumps[i]);
+ LLVMAddCase(v_switch, v_jump_coercion, b_item_coercion_blocks[i]);
+ }
+ }
+
+ LLVMPositionBuilderAtEnd(b, b_result_coercion_block);
+ if (jsestate->jump_eval_result_coercion >= 0)
+ LLVMBuildBr(b, opblocks[jsestate->jump_eval_result_coercion]);
+ else
+ LLVMBuildUnreachable(b);
+
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ LLVMPositionBuilderAtEnd(b, b_item_coercion_blocks[i]);
+ if (jsestate->eval_item_coercion_jumps[i] >= 0)
+ LLVMBuildBr(b, opblocks[jsestate->eval_item_coercion_jumps[i]]);
+ else
+ LLVMBuildUnreachable(b);
+ }
+
+ LLVMPositionBuilderAtEnd(b, b_done);
+ }
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+ }
+
+ case EEOP_JSONEXPR_COERCION:
+ build_EvalXFunc(b, mod, "ExecEvalJsonCoercionViaPopulateOrIO",
+ v_state, op, v_econtext);
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
+ case EEOP_JSONEXPR_COERCION_FINISH:
+ build_EvalXFunc(b, mod, "ExecEvalJsonCoercionFinish",
+ v_state, op);
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
case EEOP_AGGREF:
{
LLVMValueRef v_aggno;
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 3a4be09e50..ce155119b4 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -69,6 +69,7 @@ MinimalTupleTableSlot StructMinimalTupleTableSlot;
TupleDescData StructTupleDescData;
PlanState StructPlanState;
MinimalTupleData StructMinimalTupleData;
+JsonExprPostEvalState StructJsonExprPostEvalState;
/*
@@ -172,6 +173,9 @@ void *referenced_functions[] =
ExecEvalXmlExpr,
ExecEvalJsonConstructor,
ExecEvalJsonIsPredicate,
+ ExecEvalJsonCoercionViaPopulateOrIO,
+ ExecEvalJsonCoercionFinish,
+ ExecEvalJsonExprPath,
MakeExpandedObjectReadOnlyInternal,
slot_getmissingattrs,
slot_getsomeattrs_int,
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index c6fb571982..746ed3d1da 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -857,6 +857,39 @@ makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
return jve;
}
+/*
+ * makeJsonBehaviorClause -
+ * creates a JsonBehaviorClause node
+ */
+JsonBehaviorClause *
+makeJsonBehaviorClause(JsonBehavior *on_empty, JsonBehavior *on_error,
+ int location)
+{
+ JsonBehaviorClause *behavior = makeNode(JsonBehaviorClause);
+
+ behavior->on_empty = on_empty;
+ behavior->on_error = on_error;
+ behavior->location = location;
+
+ return behavior;
+}
+
+/*
+ * makeJsonBehavior -
+ * creates a JsonBehavior node
+ */
+JsonBehavior *
+makeJsonBehavior(JsonBehaviorType type, Node *default_expr, int location)
+{
+ JsonBehavior *behavior = makeNode(JsonBehavior);
+
+ behavior->btype = type;
+ behavior->default_expr = default_expr;
+ behavior->location = location;
+
+ return behavior;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index c03f4f23e2..8e49ef6b95 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -234,6 +234,15 @@ exprType(const Node *expr)
case T_JsonIsPredicate:
type = BOOLOID;
break;
+ case T_JsonExpr:
+ type = ((const JsonExpr *) expr)->returning->typid;
+ break;
+ case T_JsonItemCoercion:
+ type = exprType((Node *) ((const JsonItemCoercion *) expr)->coercion);
+ break;
+ case T_JsonCoercion:
+ type = exprType(((const JsonCoercion *) expr)->expr);
+ break;
case T_NullTest:
type = BOOLOID;
break;
@@ -493,6 +502,12 @@ exprTypmod(const Node *expr)
return exprTypmod((Node *) ((const JsonValueExpr *) expr)->formatted_expr);
case T_JsonConstructorExpr:
return ((const JsonConstructorExpr *) expr)->returning->typmod;
+ case T_JsonExpr:
+ return ((JsonExpr *) expr)->returning->typmod;
+ case T_JsonItemCoercion:
+ return exprTypmod((Node *) ((const JsonItemCoercion *) expr)->coercion);
+ case T_JsonCoercion:
+ return exprTypmod(((const JsonCoercion *) expr)->expr);
case T_CoerceToDomain:
return ((const CoerceToDomain *) expr)->resulttypmod;
case T_CoerceToDomainValue:
@@ -969,6 +984,22 @@ exprCollation(const Node *expr)
/* IS JSON's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) expr;
+ JsonCoercion *coercion = jexpr->result_coercion;
+
+ if (!coercion)
+ coll = InvalidOid;
+ else if (coercion->expr)
+ coll = exprCollation(coercion->expr);
+ else if (coercion->via_io || coercion->via_populate)
+ coll = coercion->collation;
+ else
+ coll = InvalidOid;
+ }
+ break;
+
case T_NullTest:
/* NullTest's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
@@ -1160,6 +1191,9 @@ exprSetCollation(Node *expr, Oid collation)
case T_CaseExpr:
((CaseExpr *) expr)->casecollid = collation;
break;
+ case T_CaseTestExpr:
+ ((CaseTestExpr *) expr)->collation = collation;
+ break;
case T_ArrayExpr:
((ArrayExpr *) expr)->array_collid = collation;
break;
@@ -1205,6 +1239,33 @@ exprSetCollation(Node *expr, Oid collation)
case T_JsonIsPredicate:
Assert(!OidIsValid(collation)); /* result is always boolean */
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) expr;
+
+ if (jexpr->result_coercion)
+ exprSetCollation((Node *) jexpr->result_coercion, collation);
+ else
+ Assert(!OidIsValid(collation)); /* result is always a
+ * json[b] type */
+ }
+ break;
+ case T_JsonItemCoercion:
+ exprSetCollation((Node *) ((JsonItemCoercion *) expr)->coercion,
+ collation);
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) expr;
+
+ if (coercion->expr)
+ exprSetCollation(coercion->expr, collation);
+ else if (coercion->via_io || coercion->via_populate)
+ coercion->collation = collation;
+ else
+ Assert(!OidIsValid(collation));
+ }
+ break;
case T_NullTest:
/* NullTest's result is boolean ... */
Assert(!OidIsValid(collation)); /* ... so never set a collation */
@@ -1508,6 +1569,15 @@ exprLocation(const Node *expr)
case T_JsonIsPredicate:
loc = ((const JsonIsPredicate *) expr)->location;
break;
+ case T_JsonExpr:
+ {
+ const JsonExpr *jsexpr = (const JsonExpr *) expr;
+
+ /* consider both function name and leftmost arg */
+ loc = leftmostLoc(jsexpr->location,
+ exprLocation(jsexpr->formatted_expr));
+ }
+ break;
case T_NullTest:
{
const NullTest *nexpr = (const NullTest *) expr;
@@ -2260,6 +2330,30 @@ expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ if (WALK(jexpr->formatted_expr))
+ return true;
+ if (WALK(jexpr->result_coercion))
+ return true;
+ if (WALK(jexpr->item_coercions))
+ return true;
+ if (WALK(jexpr->passing_values))
+ return true;
+ /* we assume walker doesn't care about passing_names */
+ if (jexpr->on_empty &&
+ WALK(jexpr->on_empty->default_expr))
+ return true;
+ if (WALK(jexpr->on_error->default_expr))
+ return true;
+ }
+ break;
+ case T_JsonItemCoercion:
+ return WALK(((JsonItemCoercion *) node)->coercion);
+ case T_JsonCoercion:
+ return WALK(((JsonCoercion *) node)->expr);
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
@@ -3259,6 +3353,46 @@ expression_tree_mutator_impl(Node *node,
return (Node *) newnode;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+ JsonExpr *newnode;
+
+ FLATCOPY(newnode, jexpr, JsonExpr);
+ MUTATE(newnode->path_spec, jexpr->path_spec, Node *);
+ MUTATE(newnode->formatted_expr, jexpr->formatted_expr, Node *);
+ MUTATE(newnode->result_coercion, jexpr->result_coercion, JsonCoercion *);
+ MUTATE(newnode->item_coercions, jexpr->item_coercions, List *);
+ MUTATE(newnode->passing_values, jexpr->passing_values, List *);
+ /* assume mutator does not care about passing_names */
+ if (newnode->on_empty)
+ MUTATE(newnode->on_empty->default_expr,
+ jexpr->on_empty->default_expr, Node *);
+ MUTATE(newnode->on_error->default_expr,
+ jexpr->on_error->default_expr, Node *);
+ return (Node *) newnode;
+ }
+ break;
+ case T_JsonItemCoercion:
+ {
+ JsonItemCoercion *item_coercion = (JsonItemCoercion *) node;
+ JsonItemCoercion *newnode;
+
+ FLATCOPY(newnode, item_coercion, JsonItemCoercion);
+ MUTATE(newnode->coercion, item_coercion->coercion, JsonCoercion *);
+ return (Node *) newnode;
+ }
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) node;
+ JsonCoercion *newnode;
+
+ FLATCOPY(newnode, coercion, JsonCoercion);
+ MUTATE(newnode->expr, coercion->expr, Node *);
+ return (Node *) newnode;
+ }
+ break;
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -3945,6 +4079,51 @@ raw_expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonArgument:
+ return WALK(((JsonArgument *) node)->val);
+ case T_JsonCommon:
+ {
+ JsonCommon *jc = (JsonCommon *) node;
+
+ if (WALK(jc->expr))
+ return true;
+ if (WALK(jc->pathspec))
+ return true;
+ if (WALK(jc->passing))
+ return true;
+ }
+ break;
+ case T_JsonBehaviorClause:
+ {
+ JsonBehaviorClause *jbc = (JsonBehaviorClause *) node;
+
+ if (WALK(jbc->on_empty))
+ return true;
+ if (WALK(jbc->on_error))
+ return true;
+ }
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *jb = (JsonBehavior *) node;
+
+ if (jb->btype == JSON_BEHAVIOR_DEFAULT &&
+ WALK(jb->default_expr))
+ return true;
+ }
+ break;
+ case T_JsonFuncExpr:
+ {
+ JsonFuncExpr *jfe = (JsonFuncExpr *) node;
+
+ if (WALK(jfe->common))
+ return true;
+ if (jfe->output && WALK(jfe->output))
+ return true;
+ if (jfe->behavior)
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index d6ceafd51c..3bb48b64d5 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -4850,7 +4850,8 @@ cost_qual_eval_walker(Node *node, cost_qual_eval_context *context)
IsA(node, SQLValueFunction) ||
IsA(node, XmlExpr) ||
IsA(node, CoerceToDomain) ||
- IsA(node, NextValueExpr))
+ IsA(node, NextValueExpr) ||
+ IsA(node, JsonExpr))
{
/* Treat all these as having cost 1 */
context->total.per_tuple += cpu_operator_cost;
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index 948c2bf06d..64fa79a6f7 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -53,6 +53,7 @@
#include "utils/fmgroids.h"
#include "utils/json.h"
#include "utils/jsonb.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/syscache.h"
@@ -412,6 +413,24 @@ contain_mutable_functions_walker(Node *node, void *context)
/* Check all subnodes */
}
+ if (IsA(node, JsonExpr))
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+ Const *cnst;
+
+ if (!IsA(jexpr->path_spec, Const))
+ return true;
+
+ cnst = castNode(Const, jexpr->path_spec);
+
+ Assert(cnst->consttype == JSONPATHOID);
+ if (cnst->constisnull)
+ return false;
+
+ return jspIsMutable(DatumGetJsonPathP(cnst->constvalue),
+ jexpr->passing_names, jexpr->passing_values);
+ }
+
if (IsA(node, SQLValueFunction))
{
/* all variants of SQLValueFunction are stable */
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c224df4ecc..1dc3300fde 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -278,6 +278,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ JsonBehavior *jsbehavior;
+ JsonBehaviorClause *jsbehaviorclause;
+ JsonQuotes js_quotes;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -650,14 +653,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_returning_clause_opt
json_name_and_value
json_aggregate_func
+ json_api_common_syntax
+ json_argument
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
+ json_arguments
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
+ json_wrapper_behavior
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <jsbehavior> json_behavior
+%type <jsbehaviorclause> json_behavior_clause_opt
+%type <js_quotes> json_quotes_clause_opt
/*
@@ -695,7 +705,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ COMMITTED COMPRESSION CONCURRENTLY CONDITIONAL CONFIGURATION CONFLICT
CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
@@ -706,8 +716,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
- EACH ELSE ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ESCAPE EVENT EXCEPT
- EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
+ EACH ELSE EMPTY_P ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ERROR_P ESCAPE
+ EVENT EXCEPT EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
EXTENSION EXTERNAL EXTRACT
FALSE_P FAMILY FETCH FILTER FINALIZE FIRST_P FLOAT_P FOLLOWING FOR
@@ -722,10 +732,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
- JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_OBJECT JSON_OBJECTAGG
- JSON_SCALAR JSON_SERIALIZE
+ JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
- KEY KEYS
+ KEY KEYS KEEP
LABEL LANGUAGE LARGE_P LAST_P LATERAL_P
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
@@ -739,7 +749,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
- OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR
+ OBJECT_P OF OFF OFFSET OIDS OLD OMIT ON ONLY OPERATOR OPTION OPTIONS OR
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
@@ -748,7 +758,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
- QUOTE
+ QUOTE QUOTES
RANGE READ REAL REASSIGN RECHECK RECURSIVE REF_P REFERENCES REFERENCING
REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA
@@ -759,7 +769,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
SEQUENCE SEQUENCES
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
- START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
+ START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
@@ -767,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TREAT TRIGGER TRIM TRUE_P
TRUNCATE TRUSTED TYPE_P TYPES_P
- UESCAPE UNBOUNDED UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
+ UESCAPE UNBOUNDED UNCONDITIONAL UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
UNLISTEN UNLOGGED UNTIL UPDATE USER USING
VACUUM VALID VALIDATE VALIDATOR VALUE_P VALUES VARCHAR VARIADIC VARYING
@@ -15755,6 +15765,60 @@ func_expr_common_subexpr:
n->location = @1;
$$ = (Node *) n;
}
+ | JSON_QUERY '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_QUERY_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->wrapper = $5;
+ if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@6)));
+ n->quotes = $6;
+ n->behavior = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_EXISTS '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *p = makeNode(JsonFuncExpr);
+
+ p->op = JSON_EXISTS_OP;
+ p->common = (JsonCommon *) $3;
+ p->output = (JsonOutput *) $4;
+ p->behavior = $5;
+ p->location = @1;
+ $$ = (Node *) p;
+ }
+ | JSON_VALUE '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_VALUE_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->behavior = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
;
@@ -16481,6 +16545,72 @@ opt_asymmetric: ASYMMETRIC
;
/* SQL/JSON support */
+json_api_common_syntax:
+ json_value_expr ',' a_expr /* i.e. a json_path */
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->passing = NULL;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | json_value_expr ',' a_expr AS name
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->passing = NULL;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+
+ | json_value_expr ',' a_expr /* i.e. a json_path */
+ PASSING json_arguments
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->passing = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | json_value_expr ',' a_expr AS name
+ PASSING json_arguments
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->passing = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_arguments:
+ json_argument { $$ = list_make1($1); }
+ | json_arguments ',' json_argument { $$ = lappend($1, $3); }
+ ;
+
+json_argument:
+ json_value_expr AS ColLabel
+ {
+ JsonArgument *n = makeNode(JsonArgument);
+
+ n->val = (JsonValueExpr *) $1;
+ n->name = $3;
+ $$ = (Node *) n;
+ }
+ ;
+
json_value_expr:
a_expr json_format_clause_opt
{
@@ -16506,6 +16636,27 @@ json_encoding_clause_opt:
| /* EMPTY */ { $$ = JS_ENC_DEFAULT; }
;
+/* ARRAY is a noise word */
+json_wrapper_behavior:
+ WITHOUT WRAPPER { $$ = JSW_NONE; }
+ | WITHOUT ARRAY WRAPPER { $$ = JSW_NONE; }
+ | WITH WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL ARRAY WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | /* empty */ { $$ = JSW_NONE; }
+ ;
+
+json_quotes_clause_opt:
+ KEEP QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_KEEP; }
+ | KEEP QUOTES { $$ = JS_QUOTES_KEEP; }
+ | OMIT QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_OMIT; }
+ | OMIT QUOTES { $$ = JS_QUOTES_OMIT; }
+ | /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -16519,6 +16670,30 @@ json_returning_clause_opt:
| /* EMPTY */ { $$ = NULL; }
;
+json_behavior:
+ DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
+ | ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
+ | TRUE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
+ | FALSE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
+ | UNKNOWN { $$ = makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
+ | EMPTY_P ARRAY { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P OBJECT_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
+ /* non-standard, for Oracle compatibility only */
+ | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ ;
+
+json_behavior_clause_opt:
+ json_behavior ON EMPTY_P
+ { $$ = makeJsonBehaviorClause($1, NULL, @1); }
+ | json_behavior ON ERROR_P
+ { $$ = makeJsonBehaviorClause(NULL, $1, @1); }
+ | json_behavior ON EMPTY_P json_behavior ON ERROR_P
+ { $$ = makeJsonBehaviorClause($1, $4, @1); }
+ | /* EMPTY */
+ { $$ = NULL; }
+ ;
+
json_predicate_type_constraint:
JSON { $$ = JS_TYPE_ANY; }
| JSON VALUE_P { $$ = JS_TYPE_ANY; }
@@ -17108,6 +17283,7 @@ unreserved_keyword:
| COMMIT
| COMMITTED
| COMPRESSION
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17144,10 +17320,12 @@ unreserved_keyword:
| DOUBLE_P
| DROP
| EACH
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17197,6 +17375,7 @@ unreserved_keyword:
| INSTEAD
| INVOKER
| ISOLATION
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17243,6 +17422,7 @@ unreserved_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| OPERATOR
| OPTION
| OPTIONS
@@ -17273,6 +17453,7 @@ unreserved_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REASSIGN
@@ -17332,6 +17513,7 @@ unreserved_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUPPORT
@@ -17354,6 +17536,7 @@ unreserved_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNKNOWN
| UNLISTEN
@@ -17414,10 +17597,13 @@ col_name_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
| LEAST
| NATIONAL
| NCHAR
@@ -17650,6 +17836,7 @@ bare_label_keyword:
| COMMITTED
| COMPRESSION
| CONCURRENTLY
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17702,11 +17889,13 @@ bare_label_keyword:
| DROP
| EACH
| ELSE
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| END_P
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17776,10 +17965,14 @@ bare_label_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17840,6 +18033,7 @@ bare_label_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| ONLY
| OPERATOR
| OPTION
@@ -17877,6 +18071,7 @@ bare_label_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REAL
@@ -17945,6 +18140,7 @@ bare_label_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUBSTRING
@@ -17979,6 +18175,7 @@ bare_label_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNIQUE
| UNKNOWN
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64c582c344..64571562c5 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -90,6 +90,23 @@ static Node *transformJsonParseExpr(ParseState *pstate, JsonParseExpr *expr);
static Node *transformJsonScalarExpr(ParseState *pstate, JsonScalarExpr *expr);
static Node *transformJsonSerializeExpr(ParseState *pstate,
JsonSerializeExpr *expr);
+static Node *transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *p);
+static JsonExpr *transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName);
+static void transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names);
+static JsonCoercion *coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr);
+static Oid JsonFuncExprDefaultReturnType(JsonExpr *jsexpr);
+static JsonCoercion *coerceJsonExpr(ParseState *pstate, Node *expr,
+ const JsonReturning *returning);
+static List *InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId);
+static JsonBehavior *transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning);
+static JsonBehavior *coerceJsonBehaviorDefaultExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning);
static Node *make_row_comparison_op(ParseState *pstate, List *opname,
List *largs, List *rargs, int location);
static Node *make_row_distinct_op(ParseState *pstate, List *opname,
@@ -353,6 +370,10 @@ transformExprRecurse(ParseState *pstate, Node *expr)
result = transformJsonSerializeExpr(pstate, (JsonSerializeExpr *) expr);
break;
+ case T_JsonFuncExpr:
+ result = transformJsonFuncExpr(pstate, (JsonFuncExpr *) expr);
+ break;
+
default:
/* should not reach here */
elog(ERROR, "unrecognized node type: %d", (int) nodeTag(expr));
@@ -3229,7 +3250,7 @@ makeJsonByteaToTextConversion(Node *expr, JsonFormat *format, int location)
static Node *
transformJsonValueExpr(ParseState *pstate, const char *constructName,
JsonValueExpr *ve, JsonFormatType default_format,
- Oid targettype)
+ Oid targettype, bool isarg)
{
Node *expr = transformExprRecurse(pstate, (Node *) ve->raw_expr);
Node *rawexpr;
@@ -3261,6 +3282,35 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
else
format = ve->format->format_type;
}
+ else if (isarg)
+ {
+ /* Pass SQL/JSON item types directly without conversion to json[b]. */
+ switch (exprtype)
+ {
+ case TEXTOID:
+ case NUMERICOID:
+ case BOOLOID:
+ case INT2OID:
+ case INT4OID:
+ case INT8OID:
+ case FLOAT4OID:
+ case FLOAT8OID:
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ return expr;
+
+ default:
+ if (typcategory == TYPCATEGORY_STRING)
+ return expr;
+ /* else convert argument to json[b] type */
+ break;
+ }
+
+ format = default_format;
+ }
else if (exprtype == JSONOID || exprtype == JSONBOID)
format = JS_FORMAT_DEFAULT; /* do not format json[b] types */
else
@@ -3272,7 +3322,8 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
Node *coerced;
bool only_allow_cast = OidIsValid(targettype);
- if (!only_allow_cast &&
+ if (!isarg &&
+ !only_allow_cast &&
exprtype != BYTEAOID && typcategory != TYPCATEGORY_STRING)
ereport(ERROR,
errcode(ERRCODE_DATATYPE_MISMATCH),
@@ -3425,6 +3476,11 @@ transformJsonOutput(ParseState *pstate, const JsonOutput *output,
errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("returning SETOF types is not supported in SQL/JSON functions"));
+ if (get_typtype(ret->typid) == TYPTYPE_PSEUDO)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("returning pseudo-types is not supported in SQL/JSON functions"));
+
if (ret->format->format_type == JS_FORMAT_DEFAULT)
/* assign JSONB format when returning jsonb, or JSON format otherwise */
ret->format->format_type =
@@ -3621,7 +3677,7 @@ transformJsonObjectConstructor(ParseState *pstate, JsonObjectConstructor *ctor)
Node *val = transformJsonValueExpr(pstate, "JSON_OBJECT()",
kv->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = lappend(args, key);
args = lappend(args, val);
@@ -3808,7 +3864,7 @@ transformJsonObjectAgg(ParseState *pstate, JsonObjectAgg *agg)
val = transformJsonValueExpr(pstate, "JSON_OBJECTAGG()",
agg->arg->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = list_make2(key, val);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
@@ -3864,9 +3920,8 @@ transformJsonArrayAgg(ParseState *pstate, JsonArrayAgg *agg)
Oid aggfnoid;
Oid aggtype;
- arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()",
- agg->arg,
- JS_FORMAT_DEFAULT, InvalidOid);
+ arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()", agg->arg,
+ JS_FORMAT_DEFAULT, InvalidOid, false);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
list_make1(arg));
@@ -3913,9 +3968,8 @@ transformJsonArrayConstructor(ParseState *pstate, JsonArrayConstructor *ctor)
{
JsonValueExpr *jsval = castNode(JsonValueExpr, lfirst(lc));
Node *val = transformJsonValueExpr(pstate, "JSON_ARRAY()",
- jsval,
- JS_FORMAT_DEFAULT,
- InvalidOid);
+ jsval, JS_FORMAT_DEFAULT,
+ InvalidOid, false);
args = lappend(args, val);
}
@@ -4074,7 +4128,7 @@ transformJsonParseExpr(ParseState *pstate, JsonParseExpr *jsexpr)
* function-like CASTs.
*/
arg = transformJsonValueExpr(pstate, "JSON()", jsexpr->expr,
- JS_FORMAT_JSON, returning->typid);
+ JS_FORMAT_JSON, returning->typid, false);
}
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_PARSE, list_make1(arg), NULL,
@@ -4119,7 +4173,7 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
Node *arg = transformJsonValueExpr(pstate, "JSON_SERIALIZE()",
expr->expr,
JS_FORMAT_JSON,
- InvalidOid);
+ InvalidOid, false);
if (expr->output)
{
@@ -4153,3 +4207,478 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_SERIALIZE, list_make1(arg),
NULL, returning, false, false, expr->location);
}
+
+/*
+ * Transform JSON_VALUE, JSON_QUERY, JSON_EXISTS functions into a JsonExpr node.
+ */
+static Node *
+transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
+{
+ JsonExpr *jsexpr = NULL;
+ const char *func_name = NULL;
+ JsonBehavior *on_error = NULL,
+ *on_empty = NULL;
+
+ /*
+ * Disallow FORMAT specification in the RETURNING clause of JSON_EXISTS()
+ * and JSON_VALUE().
+ */
+ if (func->output &&
+ (func->op == JSON_VALUE_OP || func->op == JSON_EXISTS_OP))
+ {
+ JsonFormat *format = func->output->returning->format;
+
+ if (format->format_type != JS_FORMAT_DEFAULT ||
+ format->encoding != JS_ENC_DEFAULT)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot specify FORMAT in RETURNING clause of %s",
+ func->op == JSON_VALUE_OP ? "JSON_VALUE()" :
+ "JSON_EXISTS()"),
+ parser_errposition(pstate, format->location)));
+ }
+
+ if (func->behavior)
+ {
+ on_error = func->behavior->on_error;
+ on_empty = func->behavior->on_empty;
+ }
+
+ switch (func->op)
+ {
+ case JSON_EXISTS_OP:
+ func_name = "JSON_EXISTS";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ jsexpr->returning->typid = BOOLOID;
+ jsexpr->returning->typmod = -1;
+ }
+ else if (jsexpr->returning->typid != BOOLOID)
+ {
+ Node *coercion_expr;
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+ int location = exprLocation((Node *) jsexpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the
+ * result of evaluating JSON_EXISTS to the coercion
+ * expression.
+ */
+ placeholder->typeId = BOOLOID;
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ coercion_expr =
+ coerce_to_target_type(pstate, (Node *) placeholder, BOOLOID,
+ jsexpr->returning->typid,
+ jsexpr->returning->typmod,
+ COERCION_EXPLICIT,
+ COERCE_IMPLICIT_CAST,
+ location);
+
+ if (coercion_expr == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast type %s to %s",
+ format_type_be(BOOLOID),
+ format_type_be(jsexpr->returning->typid)),
+ parser_coercion_errposition(pstate, location, (Node *) jsexpr)));
+
+ if (coercion_expr != (Node *) placeholder)
+ {
+ jsexpr->result_coercion = makeNode(JsonCoercion);
+ jsexpr->result_coercion->expr = coercion_expr;
+ }
+ }
+
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_FALSE,
+ jsexpr->returning);
+ break;
+
+ case JSON_QUERY_OP:
+ func_name = "JSON_QUERY";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->wrapper = func->wrapper;
+ jsexpr->omit_quotes = (func->quotes == JS_QUOTES_OMIT);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ JsonReturning *ret = jsexpr->returning;
+
+ ret->typid = JsonFuncExprDefaultReturnType(jsexpr);
+ ret->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ jsexpr->on_empty = transformJsonBehavior(pstate, on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+
+ case JSON_VALUE_OP:
+ func_name = "JSON_VALUE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ /* Make JSON_VALUE return text by default */
+ jsexpr->returning->typid = TEXTOID;
+ jsexpr->returning->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ /*
+ * Initialize JsonCoercion nodes to coerce the scalar value
+ * returned by JsonPathValue() to the "returning" type.
+ */
+ jsexpr->item_coercions =
+ InitJsonItemCoercions(pstate, jsexpr->returning,
+ exprType(jsexpr->formatted_expr));
+
+ jsexpr->on_empty = transformJsonBehavior(pstate, on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+ }
+
+ Assert(jsexpr != NULL && jsexpr->formatted_expr != NULL);
+ if (exprType(jsexpr->formatted_expr) != JSONBOID)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("%s() is not yet implemented for the json type", func_name),
+ errhint("Try casting the argument to jsonb"),
+ parser_errposition(pstate, func->location)));
+
+ return (Node *) jsexpr;
+}
+
+/*
+ * Common code for JSON_VALUE, JSON_QUERY, JSON_EXISTS transformation
+ * into a JsonExpr node.
+ */
+static JsonExpr *
+transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName)
+{
+ JsonExpr *jsexpr = makeNode(JsonExpr);
+ Node *pathspec;
+
+ jsexpr->location = func->location;
+ jsexpr->op = func->op;
+ jsexpr->formatted_expr = transformJsonValueExpr(pstate, constructName,
+ func->common->expr,
+ JS_FORMAT_JSON,
+ InvalidOid, false);
+
+ jsexpr->format = func->common->expr->format;
+
+ /* Both set in the caller. */
+ jsexpr->result_coercion = NULL;
+ jsexpr->omit_quotes = false;
+
+ pathspec = transformExprRecurse(pstate, func->common->pathspec);
+
+ jsexpr->path_spec =
+ coerce_to_target_type(pstate, pathspec, exprType(pathspec),
+ JSONPATHOID, -1,
+ COERCION_EXPLICIT, COERCE_IMPLICIT_CAST,
+ exprLocation(pathspec));
+ if (!jsexpr->path_spec)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("JSON path expression must be of type %s, not of type %s",
+ "jsonpath", format_type_be(exprType(pathspec))),
+ parser_errposition(pstate, exprLocation(pathspec))));
+
+ /*
+ * Transform and coerce to json[b] passing arguments, whose format is
+ * determined by context item type.
+ */
+ transformJsonPassingArgs(pstate, constructName,
+ exprType(jsexpr->formatted_expr) == JSONBOID ?
+ JS_FORMAT_JSONB : JS_FORMAT_JSON,
+ func->common->passing,
+ &jsexpr->passing_values,
+ &jsexpr->passing_names);
+
+ jsexpr->returning = transformJsonOutput(pstate, func->output, false);
+
+ /* JSON_QUERY support specifying FORMAT explicitly. */
+ if (func->op != JSON_QUERY_OP)
+ {
+ jsexpr->returning->format->format_type = JS_FORMAT_DEFAULT;
+ jsexpr->returning->format->encoding = JS_ENC_DEFAULT;
+ }
+
+ return jsexpr;
+}
+
+/*
+ * Transform a JSON PASSING clause.
+ */
+static void
+transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names)
+{
+ ListCell *lc;
+
+ *passing_values = NIL;
+ *passing_names = NIL;
+
+ foreach(lc, args)
+ {
+ JsonArgument *arg = castNode(JsonArgument, lfirst(lc));
+ Node *expr = transformJsonValueExpr(pstate, constructName,
+ arg->val, format,
+ InvalidOid, true);
+
+ *passing_values = lappend(*passing_values, expr);
+ *passing_names = lappend(*passing_names, makeString(arg->name));
+ }
+}
+
+/*
+ * Transform the JSON output clause of JSON_VALUE and JSON_QUERY.
+ */
+static JsonCoercion *
+coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr)
+{
+ JsonCoercion *coercion = NULL;
+ JsonReturning *returning = jsexpr->returning;
+ Node *context_item = jsexpr->formatted_expr;
+ int default_typmod;
+ Oid default_typid;
+
+ Assert(returning);
+
+ /*
+ * Forced coercion via I/O for non-JSON types, except for JSON_QUERY()
+ * which must implement the specified QUORES or WRAPPER behavior.
+ */
+ if (jsexpr->op == JSON_QUERY_OP &&
+ returning->typid != JSONOID &&
+ returning->typid != JSONBOID &&
+ (jsexpr->omit_quotes || jsexpr->wrapper != JSW_NONE))
+ {
+ coercion = makeNode(JsonCoercion);
+ coercion->expr = NULL;
+ coercion->via_io = jsexpr->omit_quotes;
+ coercion->via_populate = jsexpr->wrapper != JSW_NONE;
+
+ return coercion;
+ }
+
+ default_typid = JsonFuncExprDefaultReturnType(jsexpr);
+ default_typmod = -1;
+ if (returning->typid != default_typid ||
+ returning->typmod != default_typmod)
+ {
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * evaluating the JSON_VALUE/QUERY jsonpath expression to the coercion
+ * function.
+ */
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ placeholder->typeId = exprType(context_item);
+ placeholder->typeMod = exprTypmod(context_item);
+ placeholder->collation = exprCollation(context_item);
+
+ Assert(placeholder->typeId == default_typid);
+ Assert(placeholder->typeMod == default_typmod);
+
+ coercion = coerceJsonExpr(pstate, (Node *) placeholder, returning);
+ }
+
+ return coercion;
+}
+
+/* Returns the default type for a given JsonExpr for a given JsonFormat. */
+static Oid
+JsonFuncExprDefaultReturnType(JsonExpr *jsexpr)
+{
+ JsonFormat *format = jsexpr->format;
+ Node *context_item = jsexpr->formatted_expr;
+
+ Assert(format);
+ if (format->format_type == JS_FORMAT_JSONB)
+ return JSONBOID;
+ else if (format->format_type == JS_FORMAT_DEFAULT &&
+ exprType(context_item) == JSONBOID)
+ return JSONBOID;
+
+ return JSONOID;
+}
+
+/*
+ * Set up a JsonCoercion node to:
+ *
+ * - coerce expression to the output returning type, or
+ * - coerce using json_populate_type() if returning type requires it, or
+ * - coerce via I/O.
+ */
+static JsonCoercion *
+coerceJsonExpr(ParseState *pstate, Node *expr, const JsonReturning *returning)
+{
+ char typtype;
+ JsonCoercion *coercion = makeNode(JsonCoercion);
+
+ coercion->expr = coerceJsonFuncExpr(pstate, expr, returning, false);
+ if (coercion->expr)
+ {
+ if (coercion->expr == expr)
+ coercion->expr = NULL;
+
+ return coercion;
+ }
+
+ typtype = get_typtype(returning->typid);
+ if (returning->typid == RECORDOID ||
+ typtype == TYPTYPE_COMPOSITE ||
+ typtype == TYPTYPE_DOMAIN ||
+ type_is_array(returning->typid))
+ coercion->via_populate = true;
+ else
+ coercion->via_io = true;
+
+ return coercion;
+}
+
+/*
+ * Initialize JsonCoercion nodes for coercing a given JSON item value produced
+ * by JSON_VALUE to the target "returning" type; also see
+ * ExecPrepareJsonItemCoercion().
+ */
+static List *
+InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId)
+{
+ List *item_coercions = NIL;
+ int i;
+ Oid typeoid;
+ struct
+ {
+ JsonItemType item_type;
+ Oid typeoid;
+ } item_types[] =
+ {
+ {JsonItemTypeNull, UNKNOWNOID},
+ {JsonItemTypeString, TEXTOID},
+ {JsonItemTypeNumeric, NUMERICOID},
+ {JsonItemTypeBoolean, BOOLOID},
+ {JsonItemTypeDate, DATEOID},
+ {JsonItemTypeTime, TIMEOID},
+ {JsonItemTypeTimetz, TIMETZOID},
+ {JsonItemTypeTimestamp, TIMESTAMPOID},
+ {JsonItemTypeTimestamptz, TIMESTAMPTZOID},
+ {JsonItemTypeComposite, contextItemTypeId},
+ {JsonItemTypeInvalid, InvalidOid}
+ };
+
+ for (i = 0; OidIsValid(typeoid = item_types[i].typeoid); i++)
+ {
+ Node *expr;
+ JsonItemCoercion *item_coercion = makeNode(JsonItemCoercion);
+
+ if (typeoid == UNKNOWNOID)
+ {
+ expr = (Node *) makeNullConst(UNKNOWNOID, -1, InvalidOid);
+ }
+ else
+ {
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * JSON_VALUE jsonpath expression to the coercion function.
+ */
+ placeholder->typeId = item_types[i].typeoid;
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ expr = (Node *) placeholder;
+ }
+
+ item_coercion->item_type = item_types[i].item_type;
+ item_coercion->coercion = coerceJsonExpr(pstate, expr, returning);
+ item_coercions = lappend(item_coercions, item_coercion);
+ }
+
+ return item_coercions;
+}
+
+/*
+ * Transform a JSON BEHAVIOR clause.
+ */
+static JsonBehavior *
+transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning)
+{
+ JsonBehaviorType behavior_type = default_behavior;
+ Node *default_expr = NULL;
+
+ if (behavior)
+ {
+ behavior_type = behavior->btype;
+ if (behavior_type == JSON_BEHAVIOR_DEFAULT)
+ default_expr = transformExprRecurse(pstate, behavior->default_expr);
+ }
+
+ behavior = makeJsonBehavior(behavior_type, default_expr, -1);
+
+ /*
+ * Also coerce the DEFAULT expression, if any, to match the returning
+ * type.
+ */
+ return coerceJsonBehaviorDefaultExpr(pstate, behavior, returning);
+}
+
+/*
+ * Coerce expression in `DEFAULT expression ON ERROR / EMPTY` to the target
+ * output type.
+ */
+static JsonBehavior *
+coerceJsonBehaviorDefaultExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning)
+{
+ Oid exprtype;
+
+ if (behavior->default_expr == NULL)
+ return behavior;
+
+ exprtype = exprType(behavior->default_expr);
+
+ behavior->default_expr =
+ coerce_to_target_type(pstate,
+ behavior->default_expr,
+ exprtype,
+ returning->typid,
+ returning->typmod,
+ COERCION_EXPLICIT,
+ COERCE_IMPLICIT_CAST,
+ exprLocation((Node *) behavior));
+
+ if (behavior->default_expr == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast DEFAULT expression of type %s to %s",
+ format_type_be(exprtype),
+ format_type_be(returning->typid)),
+ parser_errposition(pstate, exprLocation((Node *) behavior))));
+
+ return behavior;
+}
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 3bc62ac3ba..8f81624f97 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -1989,6 +1989,21 @@ FigureColnameInternal(Node *node, char **name)
/* make JSON_ARRAYAGG act like a regular function */
*name = "json_arrayagg";
return 2;
+ case T_JsonFuncExpr:
+ /* make SQL/JSON functions act like a regular function */
+ switch (((JsonFuncExpr *) node)->op)
+ {
+ case JSON_EXISTS_OP:
+ *name = "json_exists";
+ return 2;
+ case JSON_QUERY_OP:
+ *name = "json_query";
+ return 2;
+ case JSON_VALUE_OP:
+ *name = "json_value";
+ return 2;
+ }
+ break;
default:
break;
}
diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c
index 8131091f79..e5451c47a2 100644
--- a/src/backend/utils/adt/formatting.c
+++ b/src/backend/utils/adt/formatting.c
@@ -4426,6 +4426,50 @@ parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
}
}
+/*
+ * Parses the datetime format string in 'fmt_str' and returns true if it
+ * contains a timezone specifier, false if not.
+ */
+bool
+datetime_format_has_tz(const char *fmt_str)
+{
+ bool incache;
+ int fmt_len = strlen(fmt_str);
+ int result;
+ FormatNode *format;
+
+ if (fmt_len > DCH_CACHE_SIZE)
+ {
+ /*
+ * Allocate new memory if format picture is bigger than static cache
+ * and do not use cache (call parser always)
+ */
+ incache = false;
+
+ format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode));
+
+ parse_format(format, fmt_str, DCH_keywords,
+ DCH_suff, DCH_index, DCH_FLAG, NULL);
+ }
+ else
+ {
+ /*
+ * Use cache buffers
+ */
+ DCHCacheEntry *ent = DCH_cache_fetch(fmt_str, false);
+
+ incache = true;
+ format = ent->format;
+ }
+
+ result = DCH_datetime_type(format);
+
+ if (!incache)
+ pfree(format);
+
+ return result & DCH_ZONED;
+}
+
/*
* do_to_timestamp: shared code for to_timestamp and to_date
*
diff --git a/src/backend/utils/adt/jsonb.c b/src/backend/utils/adt/jsonb.c
index 6f445f5c2b..6c255402c4 100644
--- a/src/backend/utils/adt/jsonb.c
+++ b/src/backend/utils/adt/jsonb.c
@@ -2163,3 +2163,65 @@ jsonb_float8(PG_FUNCTION_ARGS)
PG_RETURN_DATUM(retValue);
}
+
+/*
+ * Construct an empty array jsonb.
+ */
+Jsonb *
+JsonbMakeEmptyArray(void)
+{
+ JsonbValue jbv;
+
+ jbv.type = jbvArray;
+ jbv.val.array.elems = NULL;
+ jbv.val.array.nElems = 0;
+ jbv.val.array.rawScalar = false;
+
+ return JsonbValueToJsonb(&jbv);
+}
+
+/*
+ * Construct an empty object jsonb.
+ */
+Jsonb *
+JsonbMakeEmptyObject(void)
+{
+ JsonbValue jbv;
+
+ jbv.type = jbvObject;
+ jbv.val.object.pairs = NULL;
+ jbv.val.object.nPairs = 0;
+
+ return JsonbValueToJsonb(&jbv);
+}
+
+/*
+ * Convert jsonb to a C-string stripping quotes from scalar strings.
+ */
+char *
+JsonbUnquote(Jsonb *jb)
+{
+ if (JB_ROOT_IS_SCALAR(jb))
+ {
+ JsonbValue v;
+
+ (void) JsonbExtractScalar(&jb->root, &v);
+
+ if (v.type == jbvString)
+ return pnstrdup(v.val.string.val, v.val.string.len);
+ else if (v.type == jbvBool)
+ return pstrdup(v.val.boolean ? "true" : "false");
+ else if (v.type == jbvNumeric)
+ return DatumGetCString(DirectFunctionCall1(numeric_out,
+ PointerGetDatum(v.val.numeric)));
+ else if (v.type == jbvNull)
+ return pstrdup("null");
+ else
+ {
+ elog(ERROR, "unrecognized jsonb value type %d", v.type);
+ return NULL;
+ }
+ }
+ else
+ return JsonbToCString(NULL, &jb->root, VARSIZE(jb));
+}
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 1574ed5985..b495aedce1 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2805,7 +2805,8 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
- if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc) ||
+ JsonContainerIsScalar(jbc))
{
populate_array_report_expected_array(ctx, ndim - 1);
/* Getting here means the error was reported softly. */
@@ -2813,8 +2814,6 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
return false;
}
- Assert(!JsonContainerIsScalar(jbc));
-
it = JsonbIteratorInit(jbc);
tok = JsonbIteratorNext(&it, &val, true);
@@ -3349,6 +3348,53 @@ populate_record_field(ColumnIOData *col,
}
}
+/* recursively populate specified type from a json/jsonb value */
+Datum
+json_populate_type(Datum json_val, Oid json_type, Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext)
+{
+ JsValue jsv = {0};
+ JsonbValue jbv;
+
+ jsv.is_json = json_type == JSONOID;
+
+ if (*isnull)
+ {
+ if (jsv.is_json)
+ jsv.val.json.str = NULL;
+ else
+ jsv.val.jsonb = NULL;
+ }
+ else if (jsv.is_json)
+ {
+ text *json = DatumGetTextPP(json_val);
+
+ jsv.val.json.str = VARDATA_ANY(json);
+ jsv.val.json.len = VARSIZE_ANY_EXHDR(json);
+ jsv.val.json.type = JSON_TOKEN_INVALID; /* not used in
+ * populate_composite() */
+ }
+ else
+ {
+ Jsonb *jsonb = DatumGetJsonbP(json_val);
+
+ jsv.val.jsonb = &jbv;
+
+ /* fill binary jsonb value pointing to jb */
+ jbv.type = jbvBinary;
+ jbv.val.binary.data = &jsonb->root;
+ jbv.val.binary.len = VARSIZE(jsonb) - VARHDRSZ;
+ }
+
+ if (!*cache)
+ *cache = MemoryContextAllocZero(mcxt, sizeof(ColumnIOData));
+
+ return populate_record_field(*cache, typid, typmod, NULL, mcxt,
+ PointerGetDatum(NULL), &jsv, isnull,
+ escontext);
+}
+
static RecordIOData *
allocate_record_info(MemoryContext mcxt, int ncolumns)
{
diff --git a/src/backend/utils/adt/jsonpath.c b/src/backend/utils/adt/jsonpath.c
index c5ba3b7f1d..8d89994c10 100644
--- a/src/backend/utils/adt/jsonpath.c
+++ b/src/backend/utils/adt/jsonpath.c
@@ -68,7 +68,9 @@
#include "libpq/pqformat.h"
#include "nodes/miscnodes.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/builtins.h"
+#include "utils/formatting.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
@@ -1110,3 +1112,256 @@ jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from, JsonPathItem *to,
return true;
}
+
+/* SQL/JSON datatype status: */
+typedef enum JsonPathDatatypeStatus
+{
+ jpdsNonDateTime, /* null, bool, numeric, string, array, object */
+ jpdsUnknownDateTime, /* unknown datetime type */
+ jpdsDateTimeZoned, /* timetz, timestamptz */
+ jpdsDateTimeNonZoned /* time, timestamp, date */
+} JsonPathDatatypeStatus;
+
+/* Context for jspIsMutableWalker() */
+typedef struct JsonPathMutableContext
+{
+ List *varnames; /* list of variable names */
+ List *varexprs; /* list of variable expressions */
+ JsonPathDatatypeStatus current; /* status of @ item */
+ bool lax; /* jsonpath is lax or strict */
+ bool mutable; /* resulting mutability status */
+} JsonPathMutableContext;
+
+/*
+ * Recursive walker for jspIsMutable()
+ */
+static JsonPathDatatypeStatus
+jspIsMutableWalker(JsonPathItem *jpi, JsonPathMutableContext *cxt)
+{
+ JsonPathItem next;
+ JsonPathDatatypeStatus status = jpdsNonDateTime;
+
+ while (!cxt->mutable)
+ {
+ JsonPathItem arg;
+ JsonPathDatatypeStatus leftStatus;
+ JsonPathDatatypeStatus rightStatus;
+
+ switch (jpi->type)
+ {
+ case jpiRoot:
+ Assert(status == jpdsNonDateTime);
+ break;
+
+ case jpiCurrent:
+ Assert(status == jpdsNonDateTime);
+ status = cxt->current;
+ break;
+
+ case jpiFilter:
+ {
+ JsonPathDatatypeStatus prevStatus = cxt->current;
+
+ cxt->current = status;
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+
+ cxt->current = prevStatus;
+ break;
+ }
+
+ case jpiVariable:
+ {
+ int32 len;
+ const char *name = jspGetString(jpi, &len);
+ ListCell *lc1;
+ ListCell *lc2;
+
+ Assert(status == jpdsNonDateTime);
+
+ forboth(lc1, cxt->varnames, lc2, cxt->varexprs)
+ {
+ String *varname = lfirst_node(String, lc1);
+ Node *varexpr = lfirst(lc2);
+
+ if (strncmp(varname->sval, name, len))
+ continue;
+
+ switch (exprType(varexpr))
+ {
+ case DATEOID:
+ case TIMEOID:
+ case TIMESTAMPOID:
+ status = jpdsDateTimeNonZoned;
+ break;
+
+ case TIMETZOID:
+ case TIMESTAMPTZOID:
+ status = jpdsDateTimeZoned;
+ break;
+
+ default:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ break;
+ }
+ break;
+ }
+
+ case jpiEqual:
+ case jpiNotEqual:
+ case jpiLess:
+ case jpiGreater:
+ case jpiLessOrEqual:
+ case jpiGreaterOrEqual:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ leftStatus = jspIsMutableWalker(&arg, cxt);
+
+ jspGetRightArg(jpi, &arg);
+ rightStatus = jspIsMutableWalker(&arg, cxt);
+
+ /*
+ * Comparison of datetime type with different timezone status
+ * is mutable.
+ */
+ if (leftStatus != jpdsNonDateTime &&
+ rightStatus != jpdsNonDateTime &&
+ (leftStatus == jpdsUnknownDateTime ||
+ rightStatus == jpdsUnknownDateTime ||
+ leftStatus != rightStatus))
+ cxt->mutable = true;
+ break;
+
+ case jpiNot:
+ case jpiIsUnknown:
+ case jpiExists:
+ case jpiPlus:
+ case jpiMinus:
+ Assert(status == jpdsNonDateTime);
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiAnd:
+ case jpiOr:
+ case jpiAdd:
+ case jpiSub:
+ case jpiMul:
+ case jpiDiv:
+ case jpiMod:
+ case jpiStartsWith:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ jspGetRightArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiIndexArray:
+ for (int i = 0; i < jpi->content.array.nelems; i++)
+ {
+ JsonPathItem from;
+ JsonPathItem to;
+
+ if (jspGetArraySubscript(jpi, &from, &to, i))
+ jspIsMutableWalker(&to, cxt);
+
+ jspIsMutableWalker(&from, cxt);
+ }
+ /* FALLTHROUGH */
+
+ case jpiAnyArray:
+ if (!cxt->lax)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiAny:
+ if (jpi->content.anybounds.first > 0)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiDatetime:
+ if (jpi->content.arg)
+ {
+ char *template;
+
+ jspGetArg(jpi, &arg);
+ if (arg.type != jpiString)
+ {
+ status = jpdsNonDateTime;
+ break; /* there will be runtime error */
+ }
+
+ template = jspGetString(&arg, NULL);
+ if (datetime_format_has_tz(template))
+ status = jpdsDateTimeZoned;
+ else
+ status = jpdsDateTimeNonZoned;
+ }
+ else
+ {
+ status = jpdsUnknownDateTime;
+ }
+ break;
+
+ case jpiLikeRegex:
+ Assert(status == jpdsNonDateTime);
+ jspInitByBuffer(&arg, jpi->base, jpi->content.like_regex.expr);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ /* literals */
+ case jpiNull:
+ case jpiString:
+ case jpiNumeric:
+ case jpiBool:
+ /* accessors */
+ case jpiKey:
+ case jpiAnyKey:
+ /* special items */
+ case jpiSubscript:
+ case jpiLast:
+ /* item methods */
+ case jpiType:
+ case jpiSize:
+ case jpiAbs:
+ case jpiFloor:
+ case jpiCeiling:
+ case jpiDouble:
+ case jpiKeyValue:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ if (!jspGetNext(jpi, &next))
+ break;
+
+ jpi = &next;
+ }
+
+ return status;
+}
+
+/*
+ * Check whether jsonpath expression is immutable or not.
+ */
+bool
+jspIsMutable(JsonPath *path, List *varnames, List *varexprs)
+{
+ JsonPathMutableContext cxt;
+ JsonPathItem jpi;
+
+ cxt.varnames = varnames;
+ cxt.varexprs = varexprs;
+ cxt.current = jpdsNonDateTime;
+ cxt.lax = (path->header & JSONPATH_LAX) != 0;
+ cxt.mutable = false;
+
+ jspInit(&jpi, path);
+ jspIsMutableWalker(&jpi, &cxt);
+
+ return cxt.mutable;
+}
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 2d0599b4aa..6c5602c64d 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -87,12 +87,16 @@ typedef struct JsonBaseObjectInfo
int id;
} JsonBaseObjectInfo;
+typedef int (*JsonPathVarCallback) (void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
+
/*
* Context of jsonpath execution.
*/
typedef struct JsonPathExecContext
{
- Jsonb *vars; /* variables to substitute into jsonpath */
+ void *vars; /* variables to substitute into jsonpath */
+ JsonPathVarCallback getVar;
JsonbValue *root; /* for $ evaluation */
JsonbValue *current; /* for @ evaluation */
JsonBaseObjectInfo baseObject; /* "base object" for .keyvalue()
@@ -174,7 +178,8 @@ typedef JsonPathBool (*JsonPathPredicateCallback) (JsonPathItem *jsp,
void *param);
typedef Numeric (*BinaryArithmFunc) (Numeric num1, Numeric num2, bool *error);
-static JsonPathExecResult executeJsonPath(JsonPath *path, Jsonb *vars,
+static JsonPathExecResult executeJsonPath(JsonPath *path, void *vars,
+ JsonPathVarCallback getVar,
Jsonb *json, bool throwErrors,
JsonValueList *result, bool useTz);
static JsonPathExecResult executeItem(JsonPathExecContext *cxt,
@@ -225,8 +230,13 @@ static JsonPathExecResult appendBoolResult(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonValueList *found, JsonPathBool res);
static void getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
JsonbValue *value);
+static int GetJsonPathVar(void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
static void getJsonPathVariable(JsonPathExecContext *cxt,
- JsonPathItem *variable, Jsonb *vars, JsonbValue *value);
+ JsonPathItem *variable, JsonbValue *value);
+static int getJsonPathVariableFromJsonb(void *varsJsonb, char *varName,
+ int varNameLen, JsonbValue *val,
+ JsonbValue *baseObject);
static int JsonbArraySize(JsonbValue *jb);
static JsonPathBool executeComparison(JsonPathItem *cmp, JsonbValue *lv,
JsonbValue *rv, void *p);
@@ -284,7 +294,8 @@ jsonb_path_exists_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- res = executeJsonPath(jp, vars, jb, !silent, NULL, tz);
+ res = executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, NULL, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -339,7 +350,8 @@ jsonb_path_match_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -417,7 +429,8 @@ jsonb_path_query_internal(FunctionCallInfo fcinfo, bool tz)
vars = PG_GETARG_JSONB_P_COPY(2);
silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
funcctx->user_fctx = JsonValueListGetList(&found);
@@ -464,7 +477,8 @@ jsonb_path_query_array_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_RETURN_JSONB_P(JsonbValueToJsonb(wrapItemsInArray(&found)));
}
@@ -495,7 +509,8 @@ jsonb_path_query_first_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
if (JsonValueListLength(&found) >= 1)
PG_RETURN_JSONB_P(JsonbValueToJsonb(JsonValueListHead(&found)));
@@ -537,8 +552,9 @@ jsonb_path_query_first_tz(PG_FUNCTION_ARGS)
* In other case it tries to find all the satisfied result items.
*/
static JsonPathExecResult
-executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
- JsonValueList *result, bool useTz)
+executeJsonPath(JsonPath *path, void *vars, JsonPathVarCallback getVar,
+ Jsonb *json, bool throwErrors, JsonValueList *result,
+ bool useTz)
{
JsonPathExecContext cxt;
JsonPathExecResult res;
@@ -550,22 +566,16 @@ executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
if (!JsonbExtractScalar(&json->root, &jbv))
JsonbInitBinary(&jbv, json);
- if (vars && !JsonContainerIsObject(&vars->root))
- {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("\"vars\" argument is not an object"),
- errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
- }
-
cxt.vars = vars;
+ cxt.getVar = getVar;
cxt.laxMode = (path->header & JSONPATH_LAX) != 0;
cxt.ignoreStructuralErrors = cxt.laxMode;
cxt.root = &jbv;
cxt.current = &jbv;
cxt.baseObject.jbc = NULL;
cxt.baseObject.id = 0;
- cxt.lastGeneratedObjectId = vars ? 2 : 1;
+ /* 1 + number of base objects in vars */
+ cxt.lastGeneratedObjectId = 1 + getVar(vars, NULL, 0, NULL, NULL);
cxt.innermostArraySize = -1;
cxt.throwErrors = throwErrors;
cxt.useTz = useTz;
@@ -2108,54 +2118,118 @@ getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
&value->val.string.len);
break;
case jpiVariable:
- getJsonPathVariable(cxt, item, cxt->vars, value);
+ getJsonPathVariable(cxt, item, value);
return;
default:
elog(ERROR, "unexpected jsonpath item type");
}
}
+/*
+ * Returns the computed value of a JSON path variable with given name.
+ */
+static int
+GetJsonPathVar(void *cxt, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject)
+{
+ JsonPathVariable *var = NULL;
+ List *vars = cxt;
+ ListCell *lc;
+ int id = 1;
+
+ if (!varName)
+ return list_length(vars);
+
+ foreach(lc, vars)
+ {
+ JsonPathVariable *curvar = lfirst(lc);
+
+ if (!strncmp(curvar->name, varName, varNameLen))
+ {
+ var = curvar;
+ break;
+ }
+
+ id++;
+ }
+
+ if (!var)
+ return -1;
+
+ if (var->isnull)
+ {
+ val->type = jbvNull;
+ return 0;
+ }
+
+ JsonItemFromDatum(var->value, var->typid, var->typmod, val);
+
+ *baseObject = *val;
+ return id;
+}
+
/*
* Get the value of variable passed to jsonpath executor
*/
static void
getJsonPathVariable(JsonPathExecContext *cxt, JsonPathItem *variable,
- Jsonb *vars, JsonbValue *value)
+ JsonbValue *value)
{
char *varName;
int varNameLength;
+ JsonbValue baseObject;
+ int baseObjectId;
+
+ Assert(variable->type == jpiVariable);
+ varName = jspGetString(variable, &varNameLength);
+
+ if (!cxt->vars ||
+ (baseObjectId = cxt->getVar(cxt->vars, varName, varNameLength, value,
+ &baseObject)) < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("could not find jsonpath variable \"%s\"",
+ pnstrdup(varName, varNameLength))));
+
+ if (baseObjectId > 0)
+ setBaseObject(cxt, &baseObject, baseObjectId);
+}
+
+static int
+getJsonPathVariableFromJsonb(void *varsJsonb, char *varName, int varNameLength,
+ JsonbValue *value, JsonbValue *baseObject)
+{
+ Jsonb *vars = varsJsonb;
JsonbValue tmp;
JsonbValue *v;
- if (!vars)
+ if (!varName)
{
- value->type = jbvNull;
- return;
+ if (vars && !JsonContainerIsObject(&vars->root))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"vars\" argument is not an object"),
+ errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
+ }
+
+ return vars ? 1 : 0; /* count of base objects */
}
- Assert(variable->type == jpiVariable);
- varName = jspGetString(variable, &varNameLength);
tmp.type = jbvString;
tmp.val.string.val = varName;
tmp.val.string.len = varNameLength;
v = findJsonbValueFromContainer(&vars->root, JB_FOBJECT, &tmp);
- if (v)
- {
- *value = *v;
- pfree(v);
- }
- else
- {
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_OBJECT),
- errmsg("could not find jsonpath variable \"%s\"",
- pnstrdup(varName, varNameLength))));
- }
+ if (!v)
+ return -1;
- JsonbInitBinary(&tmp, vars);
- setBaseObject(cxt, &tmp, 1);
+ *value = *v;
+ pfree(v);
+
+ JsonbInitBinary(baseObject, vars);
+ return 1;
}
/**************** Support functions for JsonPath execution *****************/
@@ -2812,3 +2886,240 @@ compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
return DatumGetInt32(DirectFunctionCall2(cmpfunc, val1, val2));
}
+
+/* Executor-callable JSON_EXISTS implementation */
+bool
+JsonPathExists(Datum jb, JsonPath *jp, bool *error, List *vars)
+{
+ JsonPathExecResult res = executeJsonPath(jp, vars, GetJsonPathVar,
+ DatumGetJsonbP(jb), !error, NULL,
+ true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ *error = true;
+
+ return res == jperOk;
+}
+
+/* Executor-callable JSON_QUERY implementation */
+Datum
+JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper, bool *empty,
+ bool *error, List *vars)
+{
+ JsonbValue *first;
+ bool wrap;
+ JsonValueList found = {0};
+ JsonPathExecResult res PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ res = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ {
+ *error = true;
+ *empty = false;
+ return (Datum) 0;
+ }
+
+ count = JsonValueListLength(&found);
+
+ first = count > 0 ? JsonValueListHead(&found) : NULL;
+
+ if (!first)
+ wrap = false;
+ else if (wrapper == JSW_NONE)
+ wrap = false;
+ else if (wrapper == JSW_UNCONDITIONAL)
+ wrap = true;
+ else if (wrapper == JSW_CONDITIONAL)
+ wrap = count > 1 ||
+ IsAJsonbScalar(first) ||
+ (first->type == jbvBinary &&
+ JsonContainerIsScalar(first->val.binary.data));
+ else
+ {
+ elog(ERROR, "unrecognized json wrapper %d", wrapper);
+ wrap = false;
+ }
+
+ if (wrap)
+ return JsonbPGetDatum(JsonbValueToJsonb(wrapItemsInArray(&found)));
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return (Datum) 0;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_QUERY should return singleton item without wrapper"),
+ errhint("Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.")));
+ }
+
+ if (first)
+ return JsonbPGetDatum(JsonbValueToJsonb(first));
+
+ *empty = true;
+ return PointerGetDatum(NULL);
+}
+
+/* Executor-callable JSON_VALUE implementation */
+JsonbValue *
+JsonPathValue(Datum jb, JsonPath *jp, bool *empty, bool *error, List *vars)
+{
+ JsonbValue *res;
+ JsonValueList found = {0};
+ JsonPathExecResult jper PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ jper = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(jper));
+
+ if (error && jperIsError(jper))
+ {
+ *error = true;
+ *empty = false;
+ return NULL;
+ }
+
+ count = JsonValueListLength(&found);
+
+ *empty = (count == 0);
+
+ if (*empty)
+ return NULL;
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ res = JsonValueListHead(&found);
+
+ if (res->type == jbvBinary &&
+ JsonContainerIsScalar(res->val.binary.data))
+ JsonbExtractScalar(res->val.binary.data, res);
+
+ if (!IsAJsonbScalar(res))
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_SCALAR_REQUIRED),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ if (res->type == jbvNull)
+ return NULL;
+
+ return res;
+}
+
+static void
+JsonbValueInitNumericDatum(JsonbValue *jbv, Datum num)
+{
+ jbv->type = jbvNumeric;
+ jbv->val.numeric = DatumGetNumeric(num);
+}
+
+void
+JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
+{
+ switch (typid)
+ {
+ case BOOLOID:
+ res->type = jbvBool;
+ res->val.boolean = DatumGetBool(val);
+ break;
+ case NUMERICOID:
+ JsonbValueInitNumericDatum(res, val);
+ break;
+ case INT2OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int2_numeric, val));
+ break;
+ case INT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int4_numeric, val));
+ break;
+ case INT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int8_numeric, val));
+ break;
+ case FLOAT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float4_numeric, val));
+ break;
+ case FLOAT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float8_numeric, val));
+ break;
+ case TEXTOID:
+ case VARCHAROID:
+ res->type = jbvString;
+ res->val.string.val = VARDATA_ANY(val);
+ res->val.string.len = VARSIZE_ANY_EXHDR(val);
+ break;
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ res->type = jbvDatetime;
+ res->val.datetime.value = val;
+ res->val.datetime.typid = typid;
+ res->val.datetime.typmod = typmod;
+ res->val.datetime.tz = 0;
+ break;
+ case JSONBOID:
+ {
+ JsonbValue *jbv = res;
+ Jsonb *jb = DatumGetJsonbP(val);
+
+ if (JsonContainerIsScalar(&jb->root))
+ {
+ bool result PG_USED_FOR_ASSERTS_ONLY;
+
+ result = JsonbExtractScalar(&jb->root, jbv);
+ Assert(result);
+ }
+ else
+ JsonbInitBinary(jbv, jb);
+ break;
+ }
+ case JSONOID:
+ {
+ text *txt = DatumGetTextP(val);
+ char *str = text_to_cstring(txt);
+ Jsonb *jb;
+
+ jb = DatumGetJsonbP(DirectFunctionCall1(jsonb_in,
+ CStringGetDatum(str)));
+ pfree(str);
+
+ JsonItemFromDatum(JsonbPGetDatum(jb), JSONBOID, -1, res);
+ break;
+ }
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
+ }
+}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index ed7f40f053..342a2cf6aa 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -476,6 +476,8 @@ static void get_const_expr(Const *constval, deparse_context *context,
int showtype);
static void get_const_collation(Const *constval, deparse_context *context);
static void get_json_format(JsonFormat *format, StringInfo buf);
+static void get_json_returning(JsonReturning *returning, StringInfo buf,
+ bool json_format_by_default);
static void get_json_constructor(JsonConstructorExpr *ctor,
deparse_context *context, bool showimplicit);
static void get_json_constructor_options(JsonConstructorExpr *ctor,
@@ -518,6 +520,8 @@ static char *generate_qualified_type_name(Oid typid);
static text *string_to_text(char *str);
static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
+static void get_json_path_spec(Node *path_spec, deparse_context *context,
+ bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8300,6 +8304,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_WindowFunc:
case T_FuncExpr:
case T_JsonConstructorExpr:
+ case T_JsonExpr:
/* function-like: name(..) or name[..] */
return true;
@@ -8471,6 +8476,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_GroupingFunc: /* own parentheses */
case T_WindowFunc: /* own parentheses */
case T_CaseExpr: /* other separators */
+ case T_JsonExpr: /* own parentheses */
return true;
default:
return false;
@@ -8586,6 +8592,65 @@ get_rule_expr_paren(Node *node, deparse_context *context,
appendStringInfoChar(context->buf, ')');
}
+static void
+get_json_behavior(JsonBehavior *behavior, deparse_context *context,
+ const char *on)
+{
+ /*
+ * The order of array elements must correspond to the order of
+ * JsonBehaviorType members.
+ */
+ const char *behavior_names[] =
+ {
+ " NULL",
+ " ERROR",
+ " EMPTY",
+ " TRUE",
+ " FALSE",
+ " UNKNOWN",
+ " EMPTY ARRAY",
+ " EMPTY OBJECT",
+ " DEFAULT "
+ };
+
+ if ((int) behavior->btype < 0 || behavior->btype >= lengthof(behavior_names))
+ elog(ERROR, "invalid json behavior type: %d", behavior->btype);
+
+ appendStringInfoString(context->buf, behavior_names[behavior->btype]);
+
+ if (behavior->btype == JSON_BEHAVIOR_DEFAULT)
+ get_rule_expr(behavior->default_expr, context, false);
+
+ appendStringInfo(context->buf, " ON %s", on);
+}
+
+/*
+ * get_json_expr_options
+ *
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ */
+static void
+get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
+ JsonBehaviorType default_behavior)
+{
+ if (jsexpr->op == JSON_QUERY_OP)
+ {
+ if (jsexpr->wrapper == JSW_CONDITIONAL)
+ appendStringInfo(context->buf, " WITH CONDITIONAL WRAPPER");
+ else if (jsexpr->wrapper == JSW_UNCONDITIONAL)
+ appendStringInfo(context->buf, " WITH UNCONDITIONAL WRAPPER");
+
+ if (jsexpr->omit_quotes)
+ appendStringInfo(context->buf, " OMIT QUOTES");
+ }
+
+ if (jsexpr->op != JSON_EXISTS_OP &&
+ jsexpr->on_empty->btype != default_behavior)
+ get_json_behavior(jsexpr->on_empty, context, "EMPTY");
+
+ if (jsexpr->on_error->btype != default_behavior)
+ get_json_behavior(jsexpr->on_error, context, "ERROR");
+}
/* ----------
* get_rule_expr - Parse back an expression
@@ -9745,6 +9810,7 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+
case T_JsonValueExpr:
{
JsonValueExpr *jve = (JsonValueExpr *) node;
@@ -9794,6 +9860,64 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ appendStringInfoString(buf, "JSON_EXISTS(");
+ break;
+ case JSON_QUERY_OP:
+ appendStringInfoString(buf, "JSON_QUERY(");
+ break;
+ case JSON_VALUE_OP:
+ appendStringInfoString(buf, "JSON_VALUE(");
+ break;
+ }
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_json_path_spec(jexpr->path_spec, context, showimplicit);
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoString(buf, " PASSING ");
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ get_rule_expr((Node *) lfirst(lc2), context, showimplicit);
+ appendStringInfo(buf, " AS %s",
+ ((String *) lfirst_node(String, lc1))->sval);
+ }
+ }
+
+ if (jexpr->op != JSON_EXISTS_OP ||
+ jexpr->returning->typid != BOOLOID)
+ get_json_returning(jexpr->returning, context->buf,
+ jexpr->op == JSON_QUERY_OP);
+
+ get_json_expr_options(jexpr, context,
+ jexpr->op != JSON_EXISTS_OP ?
+ JSON_BEHAVIOR_NULL :
+ JSON_BEHAVIOR_FALSE);
+
+ appendStringInfoString(buf, ")");
+ }
+ break;
+
case T_List:
{
char *sep;
@@ -9917,6 +10041,7 @@ looks_like_function(Node *node)
case T_MinMaxExpr:
case T_SQLValueFunction:
case T_XmlExpr:
+ case T_JsonExpr:
/* these are all accepted by func_expr_common_subexpr */
return true;
default:
@@ -10786,6 +10911,18 @@ get_const_collation(Const *constval, deparse_context *context)
}
}
+/*
+ * get_json_path_spec - Parse back a JSON path specification
+ */
+static void
+get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit)
+{
+ if (IsA(path_spec, Const))
+ get_const_expr((Const *) path_spec, context, -1);
+ else
+ get_rule_expr(path_spec, context, showimplicit);
+}
+
/*
* get_json_format - Parse back a JsonFormat node
*/
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index c4fd933154..f8d117141e 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -23,6 +23,8 @@ struct ExprEvalStep;
struct SubscriptingRefState;
struct ScalarArrayOpExprHashTable;
struct JsonConstructorExprState;
+struct JsonbValue;
+struct JsonExprState;
/* Bits in ExprState->flags (see also execnodes.h for public flag bits): */
/* expression's interpreter has been initialized */
@@ -240,6 +242,9 @@ typedef enum ExprEvalOp
EEOP_XMLEXPR,
EEOP_JSON_CONSTRUCTOR,
EEOP_IS_JSON,
+ EEOP_JSONEXPR_PATH,
+ EEOP_JSONEXPR_COERCION,
+ EEOP_JSONEXPR_COERCION_FINISH,
EEOP_AGGREF,
EEOP_GROUPING_FUNC,
EEOP_WINDOW_FUNC,
@@ -692,6 +697,11 @@ typedef struct ExprEvalStep
JsonIsPredicate *pred; /* original expression node */
} is_json;
+ /* for EEOP_JSONEXPR_PATH */
+ struct
+ {
+ struct JsonExprState *jsestate;
+ } jsonexpr;
} d;
} ExprEvalStep;
@@ -755,6 +765,104 @@ typedef struct JsonConstructorExprState
int nargs;
} JsonConstructorExprState;
+/*
+ * Information about the state of JsonPath* evaluation.
+ */
+typedef struct JsonExprPostEvalState
+{
+ /* Did JsonPath* evaluation cause an error? */
+ NullableDatum error;
+
+ /* Is the result of JsonPath* evaluation empty? */
+ NullableDatum empty;
+
+ /*
+ * ExecEvalJsonExprPath() will set this to the address of the step to
+ * use to coerce the result of JsonPath* evaluation to the RETURNING
+ * type. Also see the description of possible step addresses this
+ * could be set to in the definition of JsonExprState.ZZ
+ */
+#define FIELDNO_JSONEXPRPOSTEVALSTATE_JUMP_EVAL_COERCION 2
+ int jump_eval_coercion;
+ bool coercing_jsonpath_item;
+} JsonExprPostEvalState;
+
+/* State for evaluating a JsonExpr, too big to inline */
+typedef struct JsonExprState
+{
+ /* original expression node */
+ JsonExpr *jsexpr;
+
+ /* value/isnull for formatted_expr */
+ NullableDatum formatted_expr;
+
+ /* value/isnull for pathspec */
+ NullableDatum pathspec;
+
+ /* JsonPathVariable entries for passing_values */
+ List *args;
+
+ /*
+ * Per-row result status info populated by ExecEvalJsonExprPath(),
+ * ExecEvalJsonCoercionViaPopulateOrIO(), and
+ * ExecEvalJsonCoercionFinish().
+ */
+ JsonExprPostEvalState post_eval;
+
+ /*
+ * Address of the step that implements the non-ERROR variant of ON ERROR
+ * and ON EMPTY behaviors, to be jumped to when ExecEvalJsonExprPath()
+ * returns false on encountering an error during JsonPath* evaluation
+ * (ON ERROR) or on finding that no matching JSON item was returned (ON
+ * EMPTY). The same steps are also performed on encountering an error
+ * when coercing JsonPath* result to the RETURNING type.
+ */
+ int jump_error;
+
+ /*
+ * Addresses of steps to perform the coercion of the JsonPath* result value
+ * to the RETURNING type. Each address points to either 1) a special
+ * EEOP_JSONEXPR_COERCION step that handles coercion using the RETURNING
+ * type's input function or by using json_via_populate(), or 2) an
+ * expression such as CoerceViaIO. It may be -1 if no coercion is
+ * necessary.
+ *
+ * jump_eval_result_coercion points to the step to evaluate the coercion
+ * given in JsonExpr.result_coercion.
+ */
+ int jump_eval_result_coercion;
+
+ /* eval_item_coercion_jumps is an array of num_item_coercions elements
+ * each containing a step address to evaluate the coercion from a value of
+ * the given JsonItemType to the RETURNING type, or -1 if no coercion is
+ * necessary. item_coercion_via_expr is an array of boolean flags of the
+ * same length that indicates whether each valid step address in the
+ * eval_item_coercion_jumps array points to an expression or a
+ * EEOP_JSONEXPR_COERCION step. ExecEvalJsonExprPath() will cause an
+ * error if it's the latter, because that mode of coercion is not
+ * supported for all JsonItemTypes.
+ */
+ int num_item_coercions;
+ int *eval_item_coercion_jumps;
+ bool *item_coercion_via_expr;
+
+ /*
+ * For passing when initializing a EEOP_IOCOERCE_SAFE step for any
+ * CoerceViaIO nodes in the expression that must be evaluated in an
+ * error-safe manner.
+ */
+ ErrorSaveContext escontext;
+
+ /* Input function info for the RETURNING type. */
+ struct
+ {
+ FmgrInfo *finfo;
+ Oid typioparam;
+ } input;
+
+ /* Cache for json_populate_type() called for coercion in some cases */
+ void *cache;
+} JsonExprState;
/* functions in execExpr.c */
extern void ExprEvalPushStep(ExprState *es, const ExprEvalStep *s);
@@ -809,6 +917,11 @@ extern void ExecEvalXmlExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalJsonConstructor(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op);
+extern void ExecEvalJsonCoercionViaPopulateOrIO(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
+extern void ExecEvalJsonCoercionFinish(ExprState *state, ExprEvalStep *op);
+extern bool ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
extern void ExecEvalGroupingFunc(ExprState *state, ExprEvalStep *op);
extern void ExecEvalSubPlan(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
diff --git a/src/include/fmgr.h b/src/include/fmgr.h
index edf61e53f3..8a6b126de1 100644
--- a/src/include/fmgr.h
+++ b/src/include/fmgr.h
@@ -85,6 +85,7 @@ typedef struct FmgrInfo
typedef struct FunctionCallInfoBaseData
{
FmgrInfo *flinfo; /* ptr to lookup info used for this call */
+#define FIELDNO_FUNCTIONCALLINFODATA_CONTEXT 1
fmNodePtr context; /* pass info about context of call */
fmNodePtr resultinfo; /* pass or return extra info about result */
Oid fncollation; /* collation for function to use */
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 3ab86de3ac..de9d8d36c5 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -85,6 +85,7 @@ extern PGDLLIMPORT LLVMTypeRef StructAggState;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerTransData;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerGroupData;
extern PGDLLIMPORT LLVMTypeRef StructPlanState;
+extern PGDLLIMPORT LLVMTypeRef StructJsonExprPostEvalState;
extern PGDLLIMPORT LLVMValueRef AttributeTemplate;
extern PGDLLIMPORT LLVMValueRef ExecEvalBoolSubroutineTemplate;
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 3180703005..783e82ac92 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -112,6 +112,10 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
int location);
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
+extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
+extern JsonBehaviorClause *makeJsonBehaviorClause(JsonBehavior *on_empty,
+ JsonBehavior *on_error,
+ int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index e494309da8..ce6893fb91 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1692,6 +1692,35 @@ typedef struct TriggerTransition
/* Nodes for SQL/JSON support */
+/*
+ * JsonQuotes -
+ * representation of [KEEP|OMIT] QUOTES clause for JSON_QUERY()
+ */
+typedef enum JsonQuotes
+{
+ JS_QUOTES_UNSPEC, /* unspecified */
+ JS_QUOTES_KEEP, /* KEEP QUOTES */
+ JS_QUOTES_OMIT /* OMIT QUOTES */
+} JsonQuotes;
+
+/*
+ * JsonBehaviorClause -
+ * representation of JSON ON ERROR / ON EMPTY clause
+ */
+typedef struct JsonBehaviorClause
+{
+ NodeTag type;
+ JsonBehavior *on_empty; /* behavior ON EMPTY */
+ JsonBehavior *on_error; /* behavior ON ERROR */
+ int location; /* token location, or -1 if unknown */
+} JsonBehaviorClause;
+
+/*
+ * JsonPathSpec -
+ * representation of JSON path constant
+ */
+typedef char *JsonPathSpec;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1703,6 +1732,47 @@ typedef struct JsonOutput
JsonReturning *returning; /* RETURNING FORMAT clause and type Oids */
} JsonOutput;
+/*
+ * JsonArgument -
+ * representation of argument from JSON PASSING clause
+ */
+typedef struct JsonArgument
+{
+ NodeTag type;
+ JsonValueExpr *val; /* argument value expression */
+ char *name; /* argument name */
+} JsonArgument;
+
+/*
+ * JsonCommon -
+ * representation of common syntax of functions using JSON path
+ */
+typedef struct JsonCommon
+{
+ NodeTag type;
+ JsonValueExpr *expr; /* context item expression */
+ Node *pathspec; /* JSON path specification expression */
+ char *pathname; /* path name, if any */
+ List *passing; /* list of PASSING clause arguments, if any */
+ int location; /* token location, or -1 if unknown */
+} JsonCommon;
+
+/*
+ * JsonFuncExpr -
+ * untransformed representation of JSON function expressions
+ */
+typedef struct JsonFuncExpr
+{
+ NodeTag type;
+ JsonExprOp op; /* expression type */
+ JsonCommon *common; /* common syntax */
+ JsonOutput *output; /* output clause, if specified */
+ JsonBehaviorClause *behavior; /* ON ERROR / EMPTY behavior, if specified */
+ JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
+ JsonQuotes quotes; /* omit or keep quotes? (JSON_QUERY only) */
+ int location; /* token location, or -1 if unknown */
+} JsonFuncExpr;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb930afb52..3f75c4608e 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -1552,6 +1552,17 @@ typedef struct XmlExpr
int location;
} XmlExpr;
+/*
+ * JsonExprOp -
+ * enumeration of JSON functions using JSON path
+ */
+typedef enum JsonExprOp
+{
+ JSON_VALUE_OP, /* JSON_VALUE() */
+ JSON_QUERY_OP, /* JSON_QUERY() */
+ JSON_EXISTS_OP /* JSON_EXISTS() */
+} JsonExprOp;
+
/*
* JsonEncoding -
* representation of JSON ENCODING clause
@@ -1576,6 +1587,37 @@ typedef enum JsonFormatType
* jsonb */
} JsonFormatType;
+/*
+ * JsonBehaviorType -
+ * enumeration of behavior types used in JSON ON ... BEHAVIOR clause
+ *
+ * If enum members are reordered, get_json_behavior() from ruleutils.c
+ * must be updated accordingly.
+ */
+typedef enum JsonBehaviorType
+{
+ JSON_BEHAVIOR_NULL = 0,
+ JSON_BEHAVIOR_ERROR,
+ JSON_BEHAVIOR_EMPTY,
+ JSON_BEHAVIOR_TRUE,
+ JSON_BEHAVIOR_FALSE,
+ JSON_BEHAVIOR_UNKNOWN,
+ JSON_BEHAVIOR_EMPTY_ARRAY,
+ JSON_BEHAVIOR_EMPTY_OBJECT,
+ JSON_BEHAVIOR_DEFAULT
+} JsonBehaviorType;
+
+/*
+ * JsonWrapper -
+ * representation of WRAPPER clause for JSON_QUERY()
+ */
+typedef enum JsonWrapper
+{
+ JSW_NONE,
+ JSW_CONDITIONAL,
+ JSW_UNCONDITIONAL,
+} JsonWrapper;
+
/*
* JsonFormat -
* representation of JSON FORMAT clause
@@ -1670,6 +1712,89 @@ typedef struct JsonIsPredicate
int location; /* token location, or -1 if unknown */
} JsonIsPredicate;
+/*
+ * JsonBehavior -
+ * representation of a given JSON behavior
+ */
+typedef struct JsonBehavior
+{
+ NodeTag type;
+ JsonBehaviorType btype; /* behavior type */
+ Node *default_expr; /* default expression when btype is
+ * JSON_BEHAVIOR_DEFAULT */
+ int location; /* token location, or -1 if unknown */
+} JsonBehavior;
+
+/*
+ * JsonItemType
+ * Represents type codes to identify a JsonCoercion node to use when
+ * coercing a given SQL/JSON items to the output SQL type
+ *
+ * The comment next to each item type mentions the JsonbValue.jbvType of the
+ * source JsonbValue value to be coerced using the expression in the
+ * JsonCoercion node.
+ *
+ * Also, see InitJsonItemCoercions() and ExecPrepareJsonItemCoercion().
+ */
+typedef enum JsonItemType
+{
+ JsonItemTypeNull = 0, /* jbvNull */
+ JsonItemTypeString = 1, /* jbvString */
+ JsonItemTypeNumeric = 2, /* jbvNumeric */
+ JsonItemTypeBoolean = 3, /* jbvBool */
+ JsonItemTypeDate = 4, /* jbvDatetime: DATEOID */
+ JsonItemTypeTime = 5, /* jbvDatetime: TIMEOID */
+ JsonItemTypeTimetz = 6, /* jbvDatetime: TIMETZOID */
+ JsonItemTypeTimestamp = 7, /* jbvDatetime: TIMESTAMPOID */
+ JsonItemTypeTimestamptz = 8, /* jbvDatetime: TIMESTAMPTZOID */
+ JsonItemTypeComposite = 9, /* jbvArray, jbvObject, jbvBinary */
+ JsonItemTypeInvalid = 10
+} JsonItemType;
+
+/*
+ * JsonCoercion -
+ * coercion from SQL/JSON item types to SQL types
+ */
+typedef struct JsonCoercion
+{
+ NodeTag type;
+ Node *expr; /* resulting expression coerced to target type */
+ bool via_populate; /* coerce result using json_populate_type()? */
+ bool via_io; /* coerce result using type input function? */
+ Oid collation; /* collation for coercion via I/O or populate */
+} JsonCoercion;
+
+typedef struct JsonItemCoercion
+{
+ NodeTag type;
+
+ JsonItemType item_type;
+ JsonCoercion *coercion;
+} JsonItemCoercion;
+
+/*
+ * JsonExpr -
+ * transformed representation of JSON_VALUE(), JSON_QUERY(), JSON_EXISTS()
+ */
+typedef struct JsonExpr
+{
+ Expr xpr;
+ JsonExprOp op; /* json function ID */
+ Node *formatted_expr; /* formatted context item expression */
+ JsonCoercion *result_coercion; /* resulting coercion to RETURNING type */
+ JsonFormat *format; /* context item format (JSON/JSONB) */
+ Node *path_spec; /* JSON path specification expression */
+ List *passing_names; /* PASSING argument names */
+ List *passing_values; /* PASSING argument values */
+ JsonReturning *returning; /* RETURNING clause type/format info */
+ JsonBehavior *on_empty; /* ON EMPTY behavior */
+ JsonBehavior *on_error; /* ON ERROR behavior */
+ List *item_coercions; /* coercions for JSON_VALUE */
+ JsonWrapper wrapper; /* WRAPPER for JSON_QUERY */
+ bool omit_quotes; /* KEEP/OMIT QUOTES for JSON_QUERY */
+ int location; /* token location, or -1 if unknown */
+} JsonExpr;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..0954d9fc7b 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -93,6 +93,7 @@ PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("conditional", CONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -147,11 +148,13 @@ PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("else", ELSE, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("empty", EMPTY_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("end", END_P, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("error", ERROR_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD, AS_LABEL)
@@ -233,10 +236,14 @@ PG_KEYWORD("join", JOIN, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json", JSON, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_array", JSON_ARRAY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_arrayagg", JSON_ARRAYAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_exists", JSON_EXISTS, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_object", JSON_OBJECT, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("keys", KEYS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -302,6 +309,7 @@ PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("omit", OMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("on", ON, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("only", ONLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -344,6 +352,7 @@ PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("quotes", QUOTES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("read", READ, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("real", REAL, COL_NAME_KEYWORD, BARE_LABEL)
@@ -414,6 +423,7 @@ PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
@@ -449,6 +459,7 @@ PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unbounded", UNBOUNDED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("unconditional", UNCONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("union", UNION, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("unique", UNIQUE, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/utils/formatting.h b/src/include/utils/formatting.h
index 0cad3a2709..9a9ae9754e 100644
--- a/src/include/utils/formatting.h
+++ b/src/include/utils/formatting.h
@@ -29,5 +29,6 @@ extern char *asc_initcap(const char *buff, size_t nbytes);
extern Datum parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
Oid *typid, int32 *typmod, int *tz,
struct Node *escontext);
+extern bool datetime_format_has_tz(const char *fmt_str);
#endif
diff --git a/src/include/utils/jsonb.h b/src/include/utils/jsonb.h
index addc9b608e..1da408a490 100644
--- a/src/include/utils/jsonb.h
+++ b/src/include/utils/jsonb.h
@@ -422,6 +422,9 @@ extern char *JsonbToCString(StringInfo out, JsonbContainer *in,
int estimated_len);
extern char *JsonbToCStringIndent(StringInfo out, JsonbContainer *in,
int estimated_len);
+extern Jsonb *JsonbMakeEmptyArray(void);
+extern Jsonb *JsonbMakeEmptyObject(void);
+extern char *JsonbUnquote(Jsonb *jb);
extern bool JsonbExtractScalar(JsonbContainer *jbc, JsonbValue *res);
extern const char *JsonbTypeName(JsonbValue *val);
diff --git a/src/include/utils/jsonfuncs.h b/src/include/utils/jsonfuncs.h
index 1c6d2be025..4c41eb5540 100644
--- a/src/include/utils/jsonfuncs.h
+++ b/src/include/utils/jsonfuncs.h
@@ -15,6 +15,7 @@
#define JSONFUNCS_H
#include "common/jsonapi.h"
+#include "nodes/nodes.h"
#include "utils/jsonb.h"
/*
@@ -87,5 +88,9 @@ extern Datum datum_to_json(Datum val, JsonTypeCategory tcategory,
extern Datum datum_to_jsonb(Datum val, JsonTypeCategory tcategory,
Oid outfuncoid);
extern Datum jsonb_from_text(text *js, bool unique_keys);
+extern Datum json_populate_type(Datum json_val, Oid json_type,
+ Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext);
#endif
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index f0181e045f..5a37133847 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -16,7 +16,9 @@
#include "fmgr.h"
#include "nodes/pg_list.h"
+#include "nodes/primnodes.h"
#include "utils/jsonb.h"
+#include "utils/jsonfuncs.h"
typedef struct
{
@@ -184,6 +186,7 @@ extern bool jspGetBool(JsonPathItem *v);
extern char *jspGetString(JsonPathItem *v, int32 *len);
extern bool jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from,
JsonPathItem *to, int i);
+extern bool jspIsMutable(JsonPath *path, List *varnames, List *varexprs);
extern const char *jspOperationName(JsonPathItemType type);
@@ -261,4 +264,28 @@ extern bool jspConvertRegexFlags(uint32 xflags, int *result,
struct Node *escontext);
+/*
+ * Evaluation of jsonpath
+ */
+
+/* External variable passed into jsonpath. */
+typedef struct JsonPathVariable
+{
+ char *name;
+ Oid typid;
+ int32 typmod;
+ Datum value;
+ bool isnull;
+} JsonPathVariable;
+
+/* SQL/JSON item */
+extern void JsonItemFromDatum(Datum val, Oid typid, int32 typmod,
+ JsonbValue *res);
+
+extern bool JsonPathExists(Datum jb, JsonPath *path, bool *error, List *vars);
+extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
+ bool *empty, bool *error, List *vars);
+extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
+ bool *error, List *vars);
+
#endif
diff --git a/src/interfaces/ecpg/preproc/ecpg.trailer b/src/interfaces/ecpg/preproc/ecpg.trailer
index 435c139ec2..b2aa44f36d 100644
--- a/src/interfaces/ecpg/preproc/ecpg.trailer
+++ b/src/interfaces/ecpg/preproc/ecpg.trailer
@@ -651,6 +651,34 @@ var_type: simple_type
$$.type_index = mm_strdup("-1");
$$.type_sizeof = NULL;
}
+ | STRING_P
+ {
+ if (INFORMIX_MODE)
+ {
+ /* In Informix mode, "string" is automatically a typedef */
+ $$.type_enum = ECPGt_string;
+ $$.type_str = mm_strdup("char");
+ $$.type_dimension = mm_strdup("-1");
+ $$.type_index = mm_strdup("-1");
+ $$.type_sizeof = NULL;
+ }
+ else
+ {
+ /* Otherwise, legal only if user typedef'ed it */
+ struct typedefs *this = get_typedef("string", false);
+
+ $$.type_str = (this->type->type_enum == ECPGt_varchar || this->type->type_enum == ECPGt_bytea) ? EMPTY : mm_strdup(this->name);
+ $$.type_enum = this->type->type_enum;
+ $$.type_dimension = this->type->type_dimension;
+ $$.type_index = this->type->type_index;
+ if (this->type->type_sizeof && strlen(this->type->type_sizeof) != 0)
+ $$.type_sizeof = this->type->type_sizeof;
+ else
+ $$.type_sizeof = cat_str(3, mm_strdup("sizeof("), mm_strdup(this->name), mm_strdup(")"));
+
+ struct_member_list[struct_level] = ECPGstruct_member_dup(this->struct_member_list);
+ }
+ }
| INTERVAL ecpg_interval
{
$$.type_enum = ECPGt_interval;
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
new file mode 100644
index 0000000000..8b87580752
--- /dev/null
+++ b/src/test/regress/expected/json_sqljson.out
@@ -0,0 +1,18 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ERROR: JSON_EXISTS() is not yet implemented for the json type
+LINE 1: SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ERROR: JSON_VALUE() is not yet implemented for the json type
+LINE 1: SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_QUERY
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ERROR: JSON_QUERY() is not yet implemented for the json type
+LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
new file mode 100644
index 0000000000..9dc71cfb38
--- /dev/null
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -0,0 +1,1095 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+ json_exists
+-------------
+
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb 'null', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+ json_exists
+-------------
+ f
+(1 row)
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+ json_exists
+-------------
+ t
+(1 row)
+
+-- extension: RETURNING clause
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING bool);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING bool);
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING int);
+ json_exists
+-------------
+ 1
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING int);
+ json_exists
+-------------
+ 0
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING text);
+ json_exists
+-------------
+ true
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING text);
+ json_exists
+-------------
+ false
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $[1]' RETURNING text FALSE ON ERROR);
+ json_exists
+-------------
+ false
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+ERROR: cannot cast type boolean to jsonb
+LINE 1: SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+ ^
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+ERROR: cannot cast type boolean to real
+LINE 1: SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+ ^
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSON); -- RETURNING FORMAT not allowed
+ERROR: cannot specify FORMAT in RETURNING clause of JSON_EXISTS()
+LINE 1: ...CT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSO...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING record);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL::jsonb, '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+ json_value
+------------
+ true
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+ json_value
+------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+ json_value
+------------
+ 123
+(1 row)
+
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+ERROR: SQL/JSON item cannot be cast to target type
+SELECT JSON_VALUE(jsonb '1.23', '$');
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "1.23"
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+ json_value
+------------
+ aa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "aaa"
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+ json_value
+------------
+ 111
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+ ?column?
+------------
+ 03-01-2017
+(1 row)
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON EMPTY ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb ERROR ON ERROR);
+ERROR: value for domain rgb violates check constraint "rgb_check"
+SELECT JSON_VALUE(jsonb '[]', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '{}', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '1', '$.a');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+ json_value
+------------
+ error
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+ json_value
+------------
+ 3
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+ json_value
+------------
+ 0
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: " "
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 5
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+ERROR: cannot specify FORMAT in RETURNING clause of JSON_VALUE()
+LINE 1: ...CT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSO...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+ x | y
+---+----
+ 0 | -2
+ 1 | 2
+ 2 | -1
+(3 rows)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point ERROR ON ERROR);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+-- Test timestamptz passing and output
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+ json_value
+--------------------------
+ Tue Feb 20 18:34:56 2018
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- JSON_QUERY
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+ json_query | json_query | json_query | json_query | json_query
+--------------------+--------------------+--------------------+----------------------+----------------------
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, null, "2"] | [1, null, "2"] | [1, null, "2"] | [[1, null, "2"]] | [[1, null, "2"]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+(6 rows)
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+ unspec | without | with cond | with uncond | with
+--------------------+--------------------+---------------------+----------------------+----------------------
+ | | | |
+ | | | |
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, 2, 3] | [1, 2, 3] | [1, 2, 3] | [[1, 2, 3]] | [[1, 2, 3]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+ | | [1, "2", null, [3]] | [1, "2", null, [3]] | [1, "2", null, [3]]
+(9 rows)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+ json_query
+------------
+ \x616161
+(1 row)
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTE...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTE...
+ ^
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_QUERY should return singleton item without wrapper
+HINT: Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+ json_query
+------------
+ [1,
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ \x7b7d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ \x7b7d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+ERROR: expected JSON array
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+ x | y | list
+---+---+--------------
+ 0 | 0 | []
+ 0 | 1 | [1]
+ 0 | 2 | [1, 2]
+ 0 | 3 | [1, 2, 3]
+ 0 | 4 | [1, 2, 3, 4]
+ 1 | 0 | []
+ 1 | 1 | [1]
+ 1 | 2 | [1, 2]
+ 1 | 3 | [1, 2, 3]
+ 1 | 4 | [1, 2, 3, 4]
+ 2 | 0 | []
+ 2 | 1 | []
+ 2 | 2 | [2]
+ 2 | 3 | [2, 3]
+ 2 | 4 | [2, 3, 4]
+ 3 | 0 | []
+ 3 | 1 | []
+ 3 | 2 | []
+ 3 | 3 | [3]
+ 3 | 4 | [3, 4]
+ 4 | 0 | []
+ 4 | 1 | []
+ 4 | 2 | []
+ 4 | 3 | []
+ 4 | 4 | [4]
+(25 rows)
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+-----------------------------------------------------
+ (1,aaa,"[1, ""2"", {}]","{""x"": [1, ""2"", {}]}",)
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+ unnest
+------------------------
+ {"a": 1, "b": ["foo"]}
+ {"a": 2, "c": {}}
+ 123
+(3 rows)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath ERROR ON ERROR);
+ERROR: syntax error at or near "{" of jsonpath input
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+--------------
+ {1,2,NULL,3}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+ json_query
+------------
+ 1
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- Test constraints
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT '12' ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+ CONSTRAINT test_jsonb_constraint6
+ CHECK (JSON_EXISTS(js::jsonb, 'strict $.a' RETURNING int TRUE ON ERROR) < 2)
+);
+\d test_jsonb_constraints
+ Table "public.test_jsonb_constraints"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+--------------------------------------------------------------------------------
+ js | text | | |
+ i | integer | | |
+ x | jsonb | | | JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+Check constraints:
+ "test_jsonb_constraint1" CHECK (js IS JSON)
+ "test_jsonb_constraint2" CHECK (JSON_EXISTS(js::jsonb, '$."a"' PASSING i + 5 AS int, i::text AS txt, ARRAY[1, 2, 3] AS arr))
+ "test_jsonb_constraint3" CHECK (JSON_VALUE(js::jsonb, '$."a"' RETURNING integer DEFAULT 12 ON EMPTY ERROR ON ERROR) > i)
+ "test_jsonb_constraint4" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ "test_jsonb_constraint5" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+ "test_jsonb_constraint6" CHECK (JSON_EXISTS(js::jsonb, 'strict $."a"' RETURNING integer TRUE ON ERROR) < 2)
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+ check_clause
+------------------------------------------------------------------------------------------------------------------------
+ (JSON_EXISTS((js)::jsonb, 'strict $."a"' RETURNING integer TRUE ON ERROR) < 2)
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ (JSON_VALUE((js)::jsonb, '$."a"' RETURNING integer DEFAULT 12 ON EMPTY ERROR ON ERROR) > i)
+ (js IS JSON)
+ JSON_EXISTS((js)::jsonb, '$."a"' PASSING (i + 5) AS int, (i)::text AS txt, ARRAY[1, 2, 3] AS arr)
+(6 rows)
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint1"
+DETAIL: Failing row contains (, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains (1, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ([], null, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ({"b": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint3"
+DETAIL: Failing row contains ({"a": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint5"
+DETAIL: Failing row contains ({"a": 7}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint4"
+DETAIL: Failing row contains ({"a": 10}, 1, [1, 2]).
+DROP TABLE test_jsonb_constraints;
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index f0987ff537..864bf04fe7 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -103,7 +103,7 @@ test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combo
# ----------
# Another group of parallel tests (JSON related)
# ----------
-test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson
+test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson json_sqljson jsonb_sqljson
# ----------
# Another group of parallel tests
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
new file mode 100644
index 0000000000..4f30fa46b9
--- /dev/null
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -0,0 +1,11 @@
+-- JSON_EXISTS
+
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+
+-- JSON_QUERY
+
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
new file mode 100644
index 0000000000..5c5d69a74e
--- /dev/null
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -0,0 +1,348 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+SELECT JSON_EXISTS(jsonb 'null', '$');
+SELECT JSON_EXISTS(jsonb '[]', '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+
+-- extension: RETURNING clause
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING bool);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING bool);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING int);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING int);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING text);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING text);
+SELECT JSON_EXISTS(jsonb '1', 'strict $[1]' RETURNING text FALSE ON ERROR);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSON); -- RETURNING FORMAT not allowed
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING record);
+
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL::jsonb, '$');
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+
+SELECT JSON_VALUE(jsonb '123', '$');
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1.23', '$');
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null);
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON EMPTY ERROR ON ERROR);
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '[]', '$');
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '{}', '$');
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1', '$.a');
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point ERROR ON ERROR);
+
+-- Test timestamptz passing and output
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- JSON_QUERY
+
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath);
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath ERROR ON ERROR);
+
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- Test constraints
+
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT '12' ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+ CONSTRAINT test_jsonb_constraint6
+ CHECK (JSON_EXISTS(js::jsonb, 'strict $.a' RETURNING int TRUE ON ERROR) < 2)
+);
+
+\d test_jsonb_constraints
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+
+DROP TABLE test_jsonb_constraints;
+
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 92c0003ab1..5ce706205c 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1242,6 +1242,7 @@ Join
JoinCostWorkspace
JoinDomain
JoinExpr
+JsonFuncExpr
JoinHashEntry
JoinPath
JoinPathExtraData
@@ -1252,18 +1253,30 @@ JsObject
JsValue
JsonAggConstructor
JsonAggState
+JsonArgument
JsonArrayAgg
JsonArrayConstructor
JsonArrayQueryConstructor
JsonBaseObjectInfo
+JsonBehavior
+JsonBehaviorClause
+JsonBehaviorType
+JsonCoercion
+JsonCoercionState
+JsonCommon
JsonConstructorExpr
JsonConstructorExprState
JsonConstructorType
JsonEncoding
+JsonExpr
+JsonExprOp
+JsonExprPostEvalState
+JsonExprState
JsonFormat
JsonFormatType
JsonHashEntry
JsonIsPredicate
+JsonItemType
JsonIterateStringValuesAction
JsonKeyValue
JsonLexContext
@@ -1281,6 +1294,7 @@ JsonParseContext
JsonParseErrorType
JsonPath
JsonPathBool
+JsonPathDatatypeStatus
JsonPathExecContext
JsonPathExecResult
JsonPathGinAddPathItemFunc
@@ -1293,10 +1307,15 @@ JsonPathGinPathItem
JsonPathItem
JsonPathItemType
JsonPathKeyword
+JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
JsonPathString
+JsonPathVarCallback
+JsonPathVariable
+JsonPathVariableEvalContext
+JsonQuotes
JsonReturning
JsonScalarExpr
JsonSemAction
@@ -1313,6 +1332,7 @@ JsonValueExpr
JsonValueList
JsonValueListIterator
JsonValueType
+JsonWrapper
Jsonb
JsonbAggState
JsonbContainer
--
2.35.3
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-11-15 17:11 ` Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
1 sibling, 1 reply; 94+ messages in thread
From: Andres Freund @ 2023-11-15 17:11 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
Hi,
Thanks, this looks like a substantial improvement. I don't quite have time to
look right now, but I thought I'd answer one question below.
On 2023-11-15 22:00:41 +0900, Amit Langote wrote:
> > This causes a nontrivial increase in the size of the parser (~5% in an
> > optimized build here), I wonder if we can do better.
>
> Hmm, sorry if I sound ignorant but what do you mean by the parser here?
gram.o, in an optimized build.
> I can see that the byte-size of gram.o increases by 1.66% after the
> above additions (1.72% with previous versions).
I'm not sure anymore how I measured it, but if you just looked at the total
file size, that might not show the full gain, because of debug symbols
etc. You can use the size command to look at just the code and data size.
> I've also checked
> using log_parser_stats that there isn't much slowdown in the
> raw-parsing speed.
What does "isn't much slowdown" mean in numbers?
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
@ 2023-11-16 08:48 ` Amit Langote <[email protected]>
2023-11-17 07:27 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
0 siblings, 2 replies; 94+ messages in thread
From: Amit Langote @ 2023-11-16 08:48 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
On Thu, Nov 16, 2023 at 2:11 AM Andres Freund <[email protected]> wrote:
> On 2023-11-15 22:00:41 +0900, Amit Langote wrote:
> > > This causes a nontrivial increase in the size of the parser (~5% in an
> > > optimized build here), I wonder if we can do better.
> >
> > Hmm, sorry if I sound ignorant but what do you mean by the parser here?
>
> gram.o, in an optimized build.
>
> > I can see that the byte-size of gram.o increases by 1.66% after the
> > above additions (1.72% with previous versions).
>
> I'm not sure anymore how I measured it, but if you just looked at the total
> file size, that might not show the full gain, because of debug symbols
> etc. You can use the size command to look at just the code and data size.
$ size /tmp/gram.*
text data bss dec hex filename
661808 0 0 661808 a1930 /tmp/gram.o.unpatched
672800 0 0 672800 a4420 /tmp/gram.o.patched
That's still a 1.66% increase in the code size:
(672800 - 661808) / 661808 % = 1.66
As for gram.c, the increase is a bit larger:
$ ll /tmp/gram.*
-rw-rw-r--. 1 amit amit 2605925 Nov 16 16:18 /tmp/gram.c.unpatched
-rw-rw-r--. 1 amit amit 2709422 Nov 16 16:22 /tmp/gram.c.patched
(2709422 - 2605925) / 2605925 % = 3.97
> > I've also checked
> > using log_parser_stats that there isn't much slowdown in the
> > raw-parsing speed.
>
> What does "isn't much slowdown" mean in numbers?
Sure, the benchmark I used measured the elapsed time (using
log_parser_stats) of parsing a simple select statement (*) averaged
over 10000 repetitions of the same query performed with `psql -c`:
Unpatched: 0.000061 seconds
Patched: 0.000061 seconds
Here's a look at the perf:
Unpatched:
0.59% [.] AllocSetAlloc
0.51% [.] hash_search_with_hash_value
0.47% [.] base_yyparse
Patched:
0.63% [.] AllocSetAlloc
0.52% [.] hash_search_with_hash_value
0.44% [.] base_yyparse
(*) select 1, 2, 3 from foo where a = 1
Is there a more relevant benchmark I could use?
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-11-17 07:27 ` jian he <[email protected]>
1 sibling, 0 replies; 94+ messages in thread
From: jian he @ 2023-11-17 07:27 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
hi.
minor issues.
In transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
func.behavior->on_empty->location and
func.behavior->on_error->location are correct.
but in ExecInitJsonExpr, jsestate->jsexpr->on_empty->location is -1,
jsestate->jsexpr->on_error->location is -1.
Maybe we can preserve these on_empty, on_error token locations in
transformJsonBehavior.
some enum declaration, ending element need an extra comma?
+ /*
+ * ExecEvalJsonExprPath() will set this to the address of the step to
+ * use to coerce the result of JsonPath* evaluation to the RETURNING
+ * type. Also see the description of possible step addresses this
+ * could be set to in the definition of JsonExprState.ZZ
+ */
"ZZ", typo?
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-11-21 03:52 ` Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
1 sibling, 1 reply; 94+ messages in thread
From: Amit Langote @ 2023-11-21 03:52 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
On Nov 16, 2023, at 17:48, Amit Langote <[email protected]> wrote:
> On Thu, Nov 16, 2023 at 2:11 AM Andres Freund <[email protected]> wrote:
>> On 2023-11-15 22:00:41 +0900, Amit Langote wrote:
>>>> This causes a nontrivial increase in the size of the parser (~5% in an
>>>> optimized build here), I wonder if we can do better.
>>>
>>> Hmm, sorry if I sound ignorant but what do you mean by the parser here?
>>
>> gram.o, in an optimized build.
>>
>>> I can see that the byte-size of gram.o increases by 1.66% after the
>>> above additions (1.72% with previous versions).
>>
>> I'm not sure anymore how I measured it, but if you just looked at the total
>> file size, that might not show the full gain, because of debug symbols
>> etc. You can use the size command to look at just the code and data size.
>
> $ size /tmp/gram.*
> text data bss dec hex filename
> 661808 0 0 661808 a1930 /tmp/gram.o.unpatched
> 672800 0 0 672800 a4420 /tmp/gram.o.patched
>
> That's still a 1.66% increase in the code size:
>
> (672800 - 661808) / 661808 % = 1.66
>
> As for gram.c, the increase is a bit larger:
>
> $ ll /tmp/gram.*
> -rw-rw-r--. 1 amit amit 2605925 Nov 16 16:18 /tmp/gram.c.unpatched
> -rw-rw-r--. 1 amit amit 2709422 Nov 16 16:22 /tmp/gram.c.patched
>
> (2709422 - 2605925) / 2605925 % = 3.97
>
>>> I've also checked
>>> using log_parser_stats that there isn't much slowdown in the
>>> raw-parsing speed.
>>
>> What does "isn't much slowdown" mean in numbers?
>
> Sure, the benchmark I used measured the elapsed time (using
> log_parser_stats) of parsing a simple select statement (*) averaged
> over 10000 repetitions of the same query performed with `psql -c`:
>
> Unpatched: 0.000061 seconds
> Patched: 0.000061 seconds
>
> Here's a look at the perf:
>
> Unpatched:
> 0.59% [.] AllocSetAlloc
> 0.51% [.] hash_search_with_hash_value
> 0.47% [.] base_yyparse
>
> Patched:
> 0.63% [.] AllocSetAlloc
> 0.52% [.] hash_search_with_hash_value
> 0.44% [.] base_yyparse
>
> (*) select 1, 2, 3 from foo where a = 1
>
> Is there a more relevant benchmark I could use?
Thought I’d share a few more numbers I collected to analyze the parser size increase over releases.
* gram.o text bytes is from the output of `size gram.o`.
* compiled with -O3
version gram.o text bytes %change gram.c bytes %change
9.6 534010 - 2108984 -
10 582554 9.09 2258313 7.08
11 584596 0.35 2313475 2.44
12 590957 1.08 2341564 1.21
13 590381 -0.09 2357327 0.67
14 600707 1.74 2428841 3.03
15 633180 5.40 2495364 2.73
16 653464 3.20 2575269 3.20
17-sqljson 672800 2.95 2709422 3.97
So if we put SQL/JSON (including JSON_TABLE()) into 17, we end up with a gram.o 2.95% larger than v16, which granted is a somewhat larger bump, though also smaller than with some of recent releases.
> --
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-11-22 19:38 ` Andres Freund <[email protected]>
2023-11-23 06:55 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-24 09:32 ` Re: remaining sql/json patches Amit Langote <[email protected]>
0 siblings, 2 replies; 94+ messages in thread
From: Andres Freund @ 2023-11-22 19:38 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
Hi,
On 2023-11-21 12:52:35 +0900, Amit Langote wrote:
> version gram.o text bytes %change gram.c bytes %change
>
> 9.6 534010 - 2108984 -
> 10 582554 9.09 2258313 7.08
> 11 584596 0.35 2313475 2.44
> 12 590957 1.08 2341564 1.21
> 13 590381 -0.09 2357327 0.67
> 14 600707 1.74 2428841 3.03
> 15 633180 5.40 2495364 2.73
> 16 653464 3.20 2575269 3.20
> 17-sqljson 672800 2.95 2709422 3.97
>
> So if we put SQL/JSON (including JSON_TABLE()) into 17, we end up with a gram.o 2.95% larger than v16, which granted is a somewhat larger bump, though also smaller than with some of recent releases.
I think it's ok to increase the size if it's necessary increases - but I also
think we've been a bit careless at times, and that that has made the parser
slower. There's probably also some "infrastructure" work we could do combat
some of the growth too.
I know I triggered the use of the .c bytes and text size, but it'd probably
more sensible to look at the size of the important tables generated by bison.
I think the most relevant defines are:
#define YYLAST 117115
#define YYNTOKENS 521
#define YYNNTS 707
#define YYNRULES 3300
#define YYNSTATES 6255
#define YYMAXUTOK 758
I think a lot of the reason we end up with such a big "state transition" space
is that a single addition to e.g. col_name_keyword or unreserved_keyword
increases the state space substantially, because it adds new transitions to so
many places. We're in quadratic territory, I think. We might be able to do
some lexer hackery to avoid that, but not sure.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
@ 2023-11-23 06:55 ` jian he <[email protected]>
2023-11-23 10:46 ` Re: remaining sql/json patches jian he <[email protected]>
1 sibling, 1 reply; 94+ messages in thread
From: jian he @ 2023-11-23 06:55 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Amit Langote <[email protected]>; Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
minor issue.
maybe you can add the following after
/src/test/regress/sql/jsonb_sqljson.sql: 127.
Test coverage for ExecPrepareJsonItemCoercion function.
SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING date '2018-02-21
12:34:56 +10' AS ts returning date);
SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING time '2018-02-21
12:34:56 +10' AS ts returning time);
SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timetz '2018-02-21
12:34:56 +10' AS ts returning timetz);
SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamp '2018-02-21
12:34:56 +10' AS ts returning timestamp);
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-23 06:55 ` Re: remaining sql/json patches jian he <[email protected]>
@ 2023-11-23 10:46 ` jian he <[email protected]>
2023-11-24 08:41 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-28 02:00 ` Re: remaining sql/json patches jian he <[email protected]>
0 siblings, 2 replies; 94+ messages in thread
From: jian he @ 2023-11-23 10:46 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Amit Langote <[email protected]>; Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
+/*
+ * Evaluate or return the step address to evaluate a coercion of a JSON item
+ * to the target type. The former if the coercion must be done right away by
+ * calling the target type's input function, and for some types, by calling
+ * json_populate_type().
+ *
+ * Returns the step address to be performed next.
+ */
+void
+ExecEvalJsonCoercionViaPopulateOrIO(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext)
the comment seems not right? it does return anything. it did the evaluation.
some logic in ExecEvalJsonCoercionViaPopulateOrIO, like if
(SOFT_ERROR_OCCURRED(escontext_p)) and if
(!InputFunctionCallSafe){...}, seems validated twice,
ExecEvalJsonCoercionFinish also did it. I uncommented the following
part, and still passed the test.
/src/backend/executor/execExprInterp.c
4452: // if (SOFT_ERROR_OCCURRED(escontext_p))
4453: // {
4454: // post_eval->error.value = BoolGetDatum(true);
4455: // *op->resvalue = (Datum) 0;
4456: // *op->resnull = true;
4457: // }
4470: // post_eval->error.value = BoolGetDatum(true);
4471: // *op->resnull = true;
4472: // *op->resvalue = (Datum) 0;
4473: return;
Correct me if I'm wrong.
like in "empty array on empty empty object on error", the "empty
array" refers to constant literal '[]' the assumed data type is jsonb,
the "empty object" refers to const literal '{}', the assumed data type
is jsonb.
--these two queries will fail very early, before ExecEvalJsonExprPath.
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.a' RETURNING int4range
default '[1.1,2]' on error);
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.a' RETURNING int4range
default '[1.1,2]' on empty);
-----these four will fail later, and will call
ExecEvalJsonCoercionViaPopulateOrIO twice.
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
object on empty empty object on error);
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
array on empty empty array on error);
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
array on empty empty object on error);
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
object on empty empty array on error);
-----however these four will not fail.
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
object on error);
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
array on error);
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
array on empty);
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
object on empty);
should the last four query fail or just return null?
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-23 06:55 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-23 10:46 ` Re: remaining sql/json patches jian he <[email protected]>
@ 2023-11-24 08:41 ` jian he <[email protected]>
1 sibling, 0 replies; 94+ messages in thread
From: jian he @ 2023-11-24 08:41 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Amit Langote <[email protected]>; Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
hi.
+ /*
+ * Set information for RETURNING type's input function used by
+ * ExecEvalJsonExprCoercion().
+ */
"ExecEvalJsonExprCoercion" comment is wrong?
+ /*
+ * Step to jump to the EEOP_JSONEXPR_FINISH step skipping over item
+ * coercion steps that will be added below, if any.
+ */
"EEOP_JSONEXPR_FINISH" comment is wrong?
seems on error, on empty behavior have some issues. The following are
tests for json_value.
select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
error on error);
select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
error on empty); ---imho, this should fail?
select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
error on empty error on error);
I did some minor refactoring, please see the attached.
In transformJsonFuncExpr, only (jsexpr->result_coercion) is not null
then do InitJsonItemCoercions.
The ExecInitJsonExpr ending part is for Adjust EEOP_JUMP steps. so I
moved "Set information for RETURNING type" inside
if (jexpr->result_coercion || jexpr->omit_quotes).
there are two if (jexpr->item_coercions). so I combined them together.
Attachments:
[application/octet-stream] ExecInitJsonExpr_minor_refactor.nocfbot (3.3K, ../../CACJufxHfy96LsCoKo0MZ=Ncz9eKMqK1A=Bh+A4egG9sFWmMs6A@mail.gmail.com/2-ExecInitJsonExpr_minor_refactor.nocfbot)
download
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-23 06:55 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-23 10:46 ` Re: remaining sql/json patches jian he <[email protected]>
@ 2023-11-28 02:00 ` jian he <[email protected]>
2023-12-05 12:25 ` Re: remaining sql/json patches Amit Langote <[email protected]>
1 sibling, 1 reply; 94+ messages in thread
From: jian he @ 2023-11-28 02:00 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Amit Langote <[email protected]>; Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
On Thu, Nov 23, 2023 at 6:46 PM jian he <[email protected]> wrote:
>
> -----however these four will not fail.
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> object on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> array on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> array on empty);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> object on empty);
>
> should the last four query fail or just return null?
I refactored making the above four queries fail.
SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
object on error);
The new error is: ERROR: cannot cast DEFAULT expression of type jsonb
to int4range.
also make the following query fail, which is as expected, imho.
select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
error on empty);
Attachments:
[application/octet-stream] v1-0001-handle-key-words-empty-array-and-empty-object.nocfbot (6.8K, ../../CACJufxEtA1DvSteFYpJWm=XhsC1BkfuiDeEJbFvKNHV53_An4A@mail.gmail.com/2-v1-0001-handle-key-words-empty-array-and-empty-object.nocfbot)
download
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-23 06:55 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-23 10:46 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-28 02:00 ` Re: remaining sql/json patches jian he <[email protected]>
@ 2023-12-05 12:25 ` Amit Langote <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Amit Langote @ 2023-12-05 12:25 UTC (permalink / raw)
To: jian he <[email protected]>; +Cc: Andres Freund <[email protected]>; Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
Hi,
Thanks for the reviews. Replying to all emails here.
On Thu, Nov 23, 2023 at 3:55 PM jian he <[email protected]> wrote:
> minor issue.
> maybe you can add the following after
> /src/test/regress/sql/jsonb_sqljson.sql: 127.
> Test coverage for ExecPrepareJsonItemCoercion function.
>
> SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING date '2018-02-21
> 12:34:56 +10' AS ts returning date);
> SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING time '2018-02-21
> 12:34:56 +10' AS ts returning time);
> SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timetz '2018-02-21
> 12:34:56 +10' AS ts returning timetz);
> SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamp '2018-02-21
> 12:34:56 +10' AS ts returning timestamp);
Added, though I decided to not include the function name in the
comment and rather reworded the nearby comment a bit.
On Thu, Nov 23, 2023 at 7:47 PM jian he <[email protected]> wrote:
> +/*
> + * Evaluate or return the step address to evaluate a coercion of a JSON item
> + * to the target type. The former if the coercion must be done right away by
> + * calling the target type's input function, and for some types, by calling
> + * json_populate_type().
> + *
> + * Returns the step address to be performed next.
> + */
> +void
> +ExecEvalJsonCoercionViaPopulateOrIO(ExprState *state, ExprEvalStep *op,
> + ExprContext *econtext)
>
> the comment seems not right? it does return anything. it did the evaluation.
Fixed the comment. Actually, I've also restored the old name of the
function because of reworking coercion machinery to use a JsonCoercion
node only for cases where the coercion is performed using I/O or
json_populdate_type().
> some logic in ExecEvalJsonCoercionViaPopulateOrIO, like if
> (SOFT_ERROR_OCCURRED(escontext_p)) and if
> (!InputFunctionCallSafe){...}, seems validated twice,
> ExecEvalJsonCoercionFinish also did it. I uncommented the following
> part, and still passed the test.
> /src/backend/executor/execExprInterp.c
> 4452: // if (SOFT_ERROR_OCCURRED(escontext_p))
> 4453: // {
> 4454: // post_eval->error.value = BoolGetDatum(true);
> 4455: // *op->resvalue = (Datum) 0;
> 4456: // *op->resnull = true;
> 4457: // }
>
> 4470: // post_eval->error.value = BoolGetDatum(true);
> 4471: // *op->resnull = true;
> 4472: // *op->resvalue = (Datum) 0;
> 4473: return;
Yes, you're right. ExecEvalJsonCoercionFinish()'s check for
soft-error suffices.
> Correct me if I'm wrong.
> like in "empty array on empty empty object on error", the "empty
> array" refers to constant literal '[]' the assumed data type is jsonb,
> the "empty object" refers to const literal '{}', the assumed data type
> is jsonb.
That's correct.
> --these two queries will fail very early, before ExecEvalJsonExprPath.
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.a' RETURNING int4range
> default '[1.1,2]' on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.a' RETURNING int4range
> default '[1.1,2]' on empty);
They fail early because the user-specified DEFAULT [ON ERROR/EMPTY]
expression is coerced at parse time.
> -----these four will fail later, and will call
> ExecEvalJsonCoercionViaPopulateOrIO twice.
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> object on empty empty object on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> array on empty empty array on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> array on empty empty object on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> object on empty empty array on error);
With the latest version, you'll now get the following errors:
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty obje...
^
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty arra...
^
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty arra...
^
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty obje...
> -----however these four will not fail.
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> object on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> array on error);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> array on empty);
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> object on empty);
> should the last four query fail or just return null?
You'll get the following with the latest version:
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty obje...
^
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty arra...
^
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty arra...
^
ERROR: cannot cast behavior expression of type jsonb to int4range
LINE 1: ...RY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty obje...
On Fri, Nov 24, 2023 at 5:41 PM jian he <[email protected]> wrote:
> + /*
> + * Set information for RETURNING type's input function used by
> + * ExecEvalJsonExprCoercion().
> + */
> "ExecEvalJsonExprCoercion" comment is wrong?
Comment removed in the latest version.
> + /*
> + * Step to jump to the EEOP_JSONEXPR_FINISH step skipping over item
> + * coercion steps that will be added below, if any.
> + */
> "EEOP_JSONEXPR_FINISH" comment is wrong?
Not wrong though the wording is misleading. It's describing what will
happen at runtime -- jump after performing result_coercion to skip
over any steps that might be present between the last of the
result_coercion steps and the EEOP_JSONEXPR_FINISH step. You can see
the code that follows is adding steps for JSON_VALUE "item" coercions,
which will be skipped by performing that jump.
> seems on error, on empty behavior have some issues. The following are
> tests for json_value.
> select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
> error on error);
> select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
> error on empty); ---imho, this should fail?
> select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
> error on empty error on error);
Yes, I agree there are issues. I think these all should give an
error. So, the no-match scenario (empty=true) should give an error
both when ERROR ON EMPTY is specified and also if only ERROR ON ERROR
is specified. With the current code, ON ERROR basically overrides ON
EMPTY clause which seems wrong.
With the latest patch, you'll get the following:
select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
error on error);
ERROR: no SQL/JSON item
select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
error on empty); ---imho, this should fail?
ERROR: no SQL/JSON item
select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
error on empty error on error);
ERROR: no SQL/JSON item
> I did some minor refactoring, please see the attached.
> In transformJsonFuncExpr, only (jsexpr->result_coercion) is not null
> then do InitJsonItemCoercions.
Makes sense.
> The ExecInitJsonExpr ending part is for Adjust EEOP_JUMP steps. so I
> moved "Set information for RETURNING type" inside
> if (jexpr->result_coercion || jexpr->omit_quotes).
> there are two if (jexpr->item_coercions). so I combined them together.
This code has moved to a different place with the latest patch,
wherein I've redesigned the io/populate-based coercions.
On Tue, Nov 28, 2023 at 11:01 AM jian he <[email protected]> wrote:
> On Thu, Nov 23, 2023 at 6:46 PM jian he <[email protected]> wrote:
> >
> > -----however these four will not fail.
> > SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> > object on error);
> > SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> > array on error);
> > SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> > array on empty);
> > SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> > object on empty);
> >
> > should the last four query fail or just return null?
>
> I refactored making the above four queries fail.
> SELECT JSON_QUERY(jsonb '{"a":[3,4]}', '$.z' RETURNING int4range empty
> object on error);
> The new error is: ERROR: cannot cast DEFAULT expression of type jsonb
> to int4range.
>
> also make the following query fail, which is as expected, imho.
> select json_value(jsonb '{"a":[123.45,1]}', '$.z' returning text
> error on empty);
Agreed. I've incorporated your suggestions into the latest patch
though not using the exact code that you shared.
Attached please find the latest patches. Other than the points
mentioned above, I've made substantial changes to how JsonBehavior and
JsonCoercion nodes work.
I've attempted to trim down the JSON_TABLE grammar (0004), but this is
all I've managed so far. Among other things, I couldn't refactor the
grammar to do away with the following:
+%nonassoc NESTED
+%left PATH
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v27-0002-Add-soft-error-handling-to-populate_record_field.patch (23.2K, ../../CA+HiwqHgvd6akb4FxC7=GpmzXDAkGODXOBM+Q2g-UQ6+NK9sxw@mail.gmail.com/2-v27-0002-Add-soft-error-handling-to-populate_record_field.patch)
download | inline diff:
From ee36f9d8bd4ad1f2820d5b5b985be79448676cb3 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Wed, 22 Nov 2023 13:18:49 +0900
Subject: [PATCH v27 2/4] Add soft error handling to populate_record_field()
An uncoming patch would like the ability to call it from the
executor for some SQL/JSON expression nodes and ask to suppress any
errors that may occur.
This commit does two things mainly:
* It modifies the various interfaces internal to jsonfuncs.c to pass
the ErrorSaveContext around.
* Make necessary modifications to handle the cases where the
processing is aborted partway through various functions that take
an ErrorSaveContext when a soft error occurs.
Note that the above changes are only intended to suppress errors in
the functions in jsonfuncs.c, but not those in any external functions
that the functions in jsonfuncs.c in turn call, such as those from
arrayfuncs.c. It is assumed that the various populate_* functions
validate the data before passing those to external functions.
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/utils/adt/jsonfuncs.c | 280 ++++++++++++++++++++++--------
1 file changed, 210 insertions(+), 70 deletions(-)
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index aa37c401e5..1574ed5985 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -265,6 +265,7 @@ typedef struct PopulateArrayContext
int *dims; /* dimensions */
int *sizes; /* current dimension counters */
int ndims; /* number of dimensions */
+ Node *escontext; /* For soft-error handling */
} PopulateArrayContext;
/* state for populate_array_json() */
@@ -389,7 +390,8 @@ static JsonParseErrorType elements_array_element_end(void *state, bool isnull);
static JsonParseErrorType elements_scalar(void *state, char *token, JsonTokenType tokentype);
/* turn a json object into a hash table */
-static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname);
+static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext);
/* semantic actions for populate_array_json */
static JsonParseErrorType populate_array_object_start(void *_state);
@@ -431,37 +433,42 @@ static Datum populate_record_worker(FunctionCallInfo fcinfo, const char *funcnam
/* helper functions for populate_record[set] */
static HeapTupleHeader populate_record(TupleDesc tupdesc, RecordIOData **record_p,
HeapTupleHeader defaultval, MemoryContext mcxt,
- JsObject *obj);
+ JsObject *obj, Node *escontext);
static void get_record_type_from_argument(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
static void get_record_type_from_query(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
-static void JsValueToJsObject(JsValue *jsv, JsObject *jso);
+static bool JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext);
static Datum populate_composite(CompositeIOData *io, Oid typid,
const char *colname, MemoryContext mcxt,
- HeapTupleHeader defaultval, JsValue *jsv, bool isnull);
-static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv);
+ HeapTupleHeader defaultval, JsValue *jsv, bool *isnull,
+ Node *escontext);
+static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext);
static void prepare_column_cache(ColumnIOData *column, Oid typid, int32 typmod,
MemoryContext mcxt, bool need_scalar);
static Datum populate_record_field(ColumnIOData *col, Oid typid, int32 typmod,
const char *colname, MemoryContext mcxt, Datum defaultval,
- JsValue *jsv, bool *isnull);
+ JsValue *jsv, bool *isnull, Node *escontext);
static RecordIOData *allocate_record_info(MemoryContext mcxt, int ncolumns);
static bool JsObjectGetField(JsObject *obj, char *field, JsValue *jsv);
static void populate_recordset_record(PopulateRecordsetState *state, JsObject *obj);
-static void populate_array_json(PopulateArrayContext *ctx, char *json, int len);
-static void populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
+static bool populate_array_json(PopulateArrayContext *ctx, char *json, int len);
+static bool populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
int ndim);
static void populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim);
-static void populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
-static void populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
-static void populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
+static bool populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
+static bool populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
+static bool populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
static Datum populate_array(ArrayIOData *aio, const char *colname,
- MemoryContext mcxt, JsValue *jsv);
+ MemoryContext mcxt, JsValue *jsv,
+ bool *isnull,
+ Node *escontext);
static Datum populate_domain(DomainIOData *io, Oid typid, const char *colname,
- MemoryContext mcxt, JsValue *jsv, bool isnull);
+ MemoryContext mcxt, JsValue *jsv, bool isnull,
+ Node *escontext);
/* functions supporting jsonb_delete, jsonb_set and jsonb_concat */
static JsonbValue *IteratorConcat(JsonbIterator **it1, JsonbIterator **it2,
@@ -2484,12 +2491,12 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
if (ndim <= 0)
{
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the value of key \"%s\".", ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array")));
}
@@ -2506,13 +2513,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
appendStringInfo(&indices, "[%d]", ctx->sizes[i]);
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s of key \"%s\".",
indices.data, ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s.",
@@ -2520,8 +2527,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
}
}
-/* set the number of dimensions of the populated array when it becomes known */
-static void
+/*
+ * Validate and set ndims for populating an array with some
+ * populate_array_*() function.
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
{
int i;
@@ -2529,7 +2541,12 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
Assert(ctx->ndims <= 0);
if (ndims <= 0)
+ {
populate_array_report_expected_array(ctx, ndims);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
ctx->ndims = ndims;
ctx->dims = palloc(sizeof(int) * ndims);
@@ -2537,10 +2554,16 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
for (i = 0; i < ndims; i++)
ctx->dims[i] = -1; /* dimensions are unknown yet */
+
+ return true;
}
-/* check the populated subarray dimension */
-static void
+/*
+ * Check the populated subarray dimension
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
{
int dim = ctx->sizes[ndim]; /* current dimension counter */
@@ -2548,21 +2571,31 @@ populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
if (ctx->dims[ndim] == -1)
ctx->dims[ndim] = dim; /* assign dimension if not yet known */
else if (ctx->dims[ndim] != dim)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed JSON array"),
errdetail("Multidimensional arrays must have "
"sub-arrays with matching dimensions.")));
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
+
/* reset the current array dimension size counter */
ctx->sizes[ndim] = 0;
/* increment the parent dimension counter if it is a nested sub-array */
if (ndim > 0)
ctx->sizes[ndim - 1]++;
+
+ return true;
}
-static void
+/*
+ * Returns true if the array element value was successfully extracted from jsv
+ * and added to ctx->astate. False if an error occurred when doing so.
+ */
+static bool
populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
{
Datum element;
@@ -2573,13 +2606,18 @@ populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
ctx->aio->element_type,
ctx->aio->element_typmod,
NULL, ctx->mcxt, PointerGetDatum(NULL),
- jsv, &element_isnull);
+ jsv, &element_isnull, ctx->escontext);
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
accumArrayResult(ctx->astate, element, element_isnull,
ctx->aio->element_type, ctx->acxt);
Assert(ndim > 0);
ctx->sizes[ndim - 1]++; /* increment current dimension counter */
+
+ return true;
}
/* json object start handler for populate_array_json() */
@@ -2592,7 +2630,12 @@ populate_array_object_start(void *_state)
if (state->ctx->ndims <= 0)
populate_array_assign_ndims(state->ctx, ndim);
else if (ndim < state->ctx->ndims)
+ {
populate_array_report_expected_array(state->ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2609,7 +2652,11 @@ populate_array_array_end(void *_state)
populate_array_assign_ndims(ctx, ndim + 1);
if (ndim < ctx->ndims)
- populate_array_check_dimension(ctx, ndim);
+ {
+ /* Report if an error occurred. */
+ if (!populate_array_check_dimension(ctx, ndim))
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2667,7 +2714,9 @@ populate_array_element_end(void *_state, bool isnull)
state->element_start) * sizeof(char);
}
- populate_array_element(ctx, ndim, &jsv);
+ /* Report if an error occurred. */
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return JSON_SEM_ACTION_FAILED;
}
return JSON_SUCCESS;
@@ -2684,7 +2733,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
if (ctx->ndims <= 0)
populate_array_assign_ndims(ctx, ndim);
else if (ndim < ctx->ndims)
+ {
populate_array_report_expected_array(ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
if (ndim == ctx->ndims)
{
@@ -2697,8 +2751,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
return JSON_SUCCESS;
}
-/* parse a json array and populate array */
-static void
+/*
+ * Parse a json array and populate array
+ *
+ * Returns false if an error occurs when parsing.
+ */
+static bool
populate_array_json(PopulateArrayContext *ctx, char *json, int len)
{
PopulateArrayState state;
@@ -2716,19 +2774,25 @@ populate_array_json(PopulateArrayContext *ctx, char *json, int len)
sem.array_element_end = populate_array_element_end;
sem.scalar = populate_array_scalar;
- pg_parse_json_or_ereport(state.lex, &sem);
-
- /* number of dimensions should be already known */
- Assert(ctx->ndims > 0 && ctx->dims);
+ if (pg_parse_json_or_errsave(state.lex, &sem, ctx->escontext))
+ {
+ /* number of dimensions should be already known */
+ Assert(ctx->ndims > 0 && ctx->dims);
+ }
freeJsonLexContext(state.lex);
+
+ return !SOFT_ERROR_OCCURRED(ctx->escontext);
}
/*
* populate_array_dim_jsonb() -- Iterate recursively through jsonb sub-array
* elements and accumulate result using given ArrayBuildState.
+ *
+ * Returns false if we return partway through because of an error in a
+ * subroutine.
*/
-static void
+static bool
populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
JsonbValue *jbv, /* jsonb sub-array */
int ndim) /* current dimension */
@@ -2742,7 +2806,12 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ {
populate_array_report_expected_array(ctx, ndim - 1);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
Assert(!JsonContainerIsScalar(jbc));
@@ -2763,7 +2832,10 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
(tok == WJB_ELEM &&
(val.type != jbvBinary ||
!JsonContainerIsArray(val.val.binary.data)))))
- populate_array_assign_ndims(ctx, ndim);
+ {
+ if (!populate_array_assign_ndims(ctx, ndim))
+ return false;
+ }
jsv.is_json = false;
jsv.val.jsonb = &val;
@@ -2776,16 +2848,21 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
* it is not the innermost dimension.
*/
if (ctx->ndims > 0 && ndim >= ctx->ndims)
- populate_array_element(ctx, ndim, &jsv);
+ {
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return false;
+ }
else
{
/* populate child sub-array */
- populate_array_dim_jsonb(ctx, &val, ndim + 1);
+ if (!populate_array_dim_jsonb(ctx, &val, ndim + 1))
+ return false;
/* number of dimensions should be already known */
Assert(ctx->ndims > 0 && ctx->dims);
- populate_array_check_dimension(ctx, ndim);
+ if (!populate_array_check_dimension(ctx, ndim))
+ return false;
}
tok = JsonbIteratorNext(&it, &val, true);
@@ -2796,14 +2873,22 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
/* free iterator, iterating until WJB_DONE */
tok = JsonbIteratorNext(&it, &val, true);
Assert(tok == WJB_DONE && !it);
+
+ return true;
}
-/* recursively populate an array from json/jsonb */
+/*
+ * Recursively populate an array from json/jsonb
+ *
+ * *isnull is set to true if an error is reported during parsing.
+ */
static Datum
populate_array(ArrayIOData *aio,
const char *colname,
MemoryContext mcxt,
- JsValue *jsv)
+ JsValue *jsv,
+ bool *isnull,
+ Node *escontext)
{
PopulateArrayContext ctx;
Datum result;
@@ -2818,14 +2903,27 @@ populate_array(ArrayIOData *aio,
ctx.ndims = 0; /* unknown yet */
ctx.dims = NULL;
ctx.sizes = NULL;
+ ctx.escontext = escontext;
if (jsv->is_json)
- populate_array_json(&ctx, jsv->val.json.str,
- jsv->val.json.len >= 0 ? jsv->val.json.len
- : strlen(jsv->val.json.str));
+ {
+ /* Return null if an error was found. */
+ if (!populate_array_json(&ctx, jsv->val.json.str,
+ jsv->val.json.len >= 0 ? jsv->val.json.len
+ : strlen(jsv->val.json.str)))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
+ }
else
{
- populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1);
+ /* Return null if an error was found. */
+ if (!populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
ctx.dims[0] = ctx.sizes[0];
}
@@ -2843,11 +2941,16 @@ populate_array(ArrayIOData *aio,
pfree(ctx.sizes);
pfree(lbs);
+ *isnull = false;
return result;
}
-static void
-JsValueToJsObject(JsValue *jsv, JsObject *jso)
+/*
+ * Returns false if an error occurs, provided escontext points to an
+ * ErrorSaveContext.
+ */
+static bool
+JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext)
{
jso->is_json = jsv->is_json;
@@ -2859,7 +2962,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
jsv->val.json.len >= 0
? jsv->val.json.len
: strlen(jsv->val.json.str),
- "populate_composite");
+ "populate_composite",
+ escontext);
}
else
{
@@ -2877,7 +2981,7 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
is_scalar = IsAJsonbScalar(jbv) ||
(jbv->type == jbvBinary &&
JsonContainerIsScalar(jbv->val.binary.data));
- ereport(ERROR,
+ errsave(escontext,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
is_scalar
? errmsg("cannot call %s on a scalar",
@@ -2886,6 +2990,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
"populate_composite")));
}
}
+
+ return !SOFT_ERROR_OCCURRED(escontext);
}
/* acquire or update cached tuple descriptor for a composite type */
@@ -2912,7 +3018,12 @@ update_cached_tupdesc(CompositeIOData *io, MemoryContext mcxt)
}
}
-/* recursively populate a composite (row type) value from json/jsonb */
+/*
+ * Recursively populate a composite (row type) value from json/jsonb
+ *
+ * Returns null if an error occurs in a subroutine, provided escontext points
+ * to an ErrorSaveContext.
+ */
static Datum
populate_composite(CompositeIOData *io,
Oid typid,
@@ -2920,14 +3031,15 @@ populate_composite(CompositeIOData *io,
MemoryContext mcxt,
HeapTupleHeader defaultval,
JsValue *jsv,
- bool isnull)
+ bool *isnull,
+ Node *escontext)
{
Datum result;
/* acquire/update cached tuple descriptor */
update_cached_tupdesc(io, mcxt);
- if (isnull)
+ if (*isnull)
result = (Datum) 0;
else
{
@@ -2935,11 +3047,15 @@ populate_composite(CompositeIOData *io,
JsObject jso;
/* prepare input value */
- JsValueToJsObject(jsv, &jso);
+ if (!JsValueToJsObject(jsv, &jso, escontext))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
/* populate resulting record tuple */
tuple = populate_record(io->tupdesc, &io->record_io,
- defaultval, mcxt, &jso);
+ defaultval, mcxt, &jso, escontext);
result = HeapTupleHeaderGetDatum(tuple);
JsObjectFree(&jso);
@@ -2951,14 +3067,20 @@ populate_composite(CompositeIOData *io,
* now, we can tell by comparing typid to base_typid.)
*/
if (typid != io->base_typid && typid != RECORDOID)
- domain_check(result, isnull, typid, &io->domain_info, mcxt);
+ domain_check(result, *isnull, typid, &io->domain_info, mcxt);
return result;
}
-/* populate non-null scalar value from json/jsonb value */
+/*
+ * Populate non-null scalar value from json/jsonb value.
+ *
+ * Returns null if an error occurs during the call to type input function,
+ * provided escontext is valid.
+ */
static Datum
-populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
+populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext)
{
Datum res;
char *str = NULL;
@@ -3029,7 +3151,12 @@ populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
elog(ERROR, "unrecognized jsonb type: %d", (int) jbv->type);
}
- res = InputFunctionCall(&io->typiofunc, str, io->typioparam, typmod);
+ if (!InputFunctionCallSafe(&io->typiofunc, str, io->typioparam, typmod,
+ escontext, &res))
+ {
+ res = (Datum) 0;
+ *isnull = true;
+ }
/* free temporary buffer */
if (str != json)
@@ -3044,7 +3171,8 @@ populate_domain(DomainIOData *io,
const char *colname,
MemoryContext mcxt,
JsValue *jsv,
- bool isnull)
+ bool isnull,
+ Node *escontext)
{
Datum res;
@@ -3055,8 +3183,8 @@ populate_domain(DomainIOData *io,
res = populate_record_field(io->base_io,
io->base_typid, io->base_typmod,
colname, mcxt, PointerGetDatum(NULL),
- jsv, &isnull);
- Assert(!isnull);
+ jsv, &isnull, escontext);
+ Assert(!isnull || SOFT_ERROR_OCCURRED(escontext));
}
domain_check(res, isnull, typid, &io->domain_info, mcxt);
@@ -3160,7 +3288,8 @@ populate_record_field(ColumnIOData *col,
MemoryContext mcxt,
Datum defaultval,
JsValue *jsv,
- bool *isnull)
+ bool *isnull,
+ Node *escontext)
{
TypeCat typcat;
@@ -3193,10 +3322,12 @@ populate_record_field(ColumnIOData *col,
switch (typcat)
{
case TYPECAT_SCALAR:
- return populate_scalar(&col->scalar_io, typid, typmod, jsv);
+ return populate_scalar(&col->scalar_io, typid, typmod, jsv,
+ isnull, escontext);
case TYPECAT_ARRAY:
- return populate_array(&col->io.array, colname, mcxt, jsv);
+ return populate_array(&col->io.array, colname, mcxt, jsv,
+ isnull, escontext);
case TYPECAT_COMPOSITE:
case TYPECAT_COMPOSITE_DOMAIN:
@@ -3205,11 +3336,12 @@ populate_record_field(ColumnIOData *col,
DatumGetPointer(defaultval)
? DatumGetHeapTupleHeader(defaultval)
: NULL,
- jsv, *isnull);
+ jsv, isnull,
+ escontext);
case TYPECAT_DOMAIN:
return populate_domain(&col->io.domain, typid, colname, mcxt,
- jsv, *isnull);
+ jsv, *isnull, escontext);
default:
elog(ERROR, "unrecognized type category '%c'", typcat);
@@ -3266,7 +3398,8 @@ populate_record(TupleDesc tupdesc,
RecordIOData **record_p,
HeapTupleHeader defaultval,
MemoryContext mcxt,
- JsObject *obj)
+ JsObject *obj,
+ Node *escontext)
{
RecordIOData *record = *record_p;
Datum *values;
@@ -3358,7 +3491,8 @@ populate_record(TupleDesc tupdesc,
mcxt,
nulls[i] ? (Datum) 0 : values[i],
&field,
- &nulls[i]);
+ &nulls[i],
+ escontext);
}
res = heap_form_tuple(tupdesc, values, nulls);
@@ -3445,6 +3579,7 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
JsValue jsv = {0};
HeapTupleHeader rec;
Datum rettuple;
+ bool isnull;
JsonbValue jbv;
MemoryContext fnmcxt = fcinfo->flinfo->fn_mcxt;
PopulateRecordCache *cache = fcinfo->flinfo->fn_extra;
@@ -3531,8 +3666,11 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
jbv.val.binary.len = VARSIZE(jb) - VARHDRSZ;
}
+ isnull = false;
rettuple = populate_composite(&cache->c.io.composite, cache->argtype,
- NULL, fnmcxt, rec, &jsv, false);
+ NULL, fnmcxt, rec, &jsv, &isnull,
+ NULL);
+ Assert(!isnull);
PG_RETURN_DATUM(rettuple);
}
@@ -3543,7 +3681,8 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
* decompose a json object into a hash table.
*/
static HTAB *
-get_json_object_as_hash(char *json, int len, const char *funcname)
+get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext)
{
HASHCTL ctl;
HTAB *tab;
@@ -3572,7 +3711,7 @@ get_json_object_as_hash(char *json, int len, const char *funcname)
sem->object_field_start = hash_object_field_start;
sem->object_field_end = hash_object_field_end;
- pg_parse_json_or_ereport(state->lex, sem);
+ pg_parse_json_or_errsave(state->lex, sem, escontext);
freeJsonLexContext(state->lex);
@@ -3743,7 +3882,8 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
&cache->c.io.composite.record_io,
state->rec,
cache->fn_mcxt,
- obj);
+ obj,
+ NULL);
/* if it's domain over composite, check domain constraints */
if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
--
2.35.3
[application/octet-stream] v27-0001-Add-soft-error-handling-to-some-expression-nodes.patch (9.5K, ../../CA+HiwqHgvd6akb4FxC7=GpmzXDAkGODXOBM+Q2g-UQ6+NK9sxw@mail.gmail.com/3-v27-0001-Add-soft-error-handling-to-some-expression-nodes.patch)
download | inline diff:
From 0a4bc2d5ccedf652cd538e45a5f82035165bc5fd Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Wed, 22 Nov 2023 13:18:39 +0900
Subject: [PATCH v27 1/4] Add soft error handling to some expression nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This adjusts the CoerceViaIO and CoerceToDomain expression evaluation
code to handle errors softly.
For CoerceViaIo, this adds a new ExprEvalStep opcode
EEOP_IOCOERCE_SAFE, which is implemented in new function
ExecEvalCoerceViaIOSafe(). The only difference from EEOP_IOCOERCE's
inline implementation is that the input function receives an
ErrorSaveContext via the function's FunctionCallInfo.context, which
it can use to handle errors softly.
For CoerceToDomain, this simply entails replacing the ereport() in
ExecEvalConstraintNotNull() and ExecEvalConstraintCheck() by
errsave() passing it the ErrorSaveContext passed in the expression's
ExprEvalStep.
In both cases, the ErrorSaveContext to be used is passed by setting
ExprState.escontext to point to it before calling ExecInitExprRec()
on the expression tree whose errors are to be suppressed.
Note that no call site of ExecInitExprRec() has been changed in this
commit, so there's no functional change. This is intended for
implementing new SQL/JSON expression nodes in future commits that
will use to it suppress errors that may occur during type coercions.
Reviewed-by: Álvaro Herrera
Reviewed-by: Andres Freund
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/executor/execExpr.c | 8 ++-
src/backend/executor/execExprInterp.c | 74 ++++++++++++++++++++++++++-
src/backend/jit/llvm/llvmjit_expr.c | 6 +++
src/backend/jit/llvm/llvmjit_types.c | 1 +
src/include/executor/execExpr.h | 4 ++
src/include/nodes/execnodes.h | 7 +++
6 files changed, 97 insertions(+), 3 deletions(-)
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 2c62b0c9c8..34bd2102b5 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -1563,7 +1563,10 @@ ExecInitExprRec(Expr *node, ExprState *state,
* We don't check permissions here as a type's input/output
* function are assumed to be executable by everyone.
*/
- scratch.opcode = EEOP_IOCOERCE;
+ if (state->escontext == NULL)
+ scratch.opcode = EEOP_IOCOERCE;
+ else
+ scratch.opcode = EEOP_IOCOERCE_SAFE;
/* lookup the source type's output function */
scratch.d.iocoerce.finfo_out = palloc0(sizeof(FmgrInfo));
@@ -1599,6 +1602,8 @@ ExecInitExprRec(Expr *node, ExprState *state,
fcinfo_in->args[2].value = Int32GetDatum(-1);
fcinfo_in->args[2].isnull = false;
+ fcinfo_in->context = (Node *) state->escontext;
+
ExprEvalPushStep(state, &scratch);
break;
}
@@ -3306,6 +3311,7 @@ ExecInitCoerceToDomain(ExprEvalStep *scratch, CoerceToDomain *ctest,
/* we'll allocate workspace only if needed */
scratch->d.domaincheck.checkvalue = NULL;
scratch->d.domaincheck.checknull = NULL;
+ scratch->d.domaincheck.escontext = state->escontext;
/*
* Evaluate argument - it's fine to directly store it into resv/resnull,
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 24c2b60c62..d5db96444c 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -63,6 +63,7 @@
#include "executor/nodeSubplan.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/miscnodes.h"
#include "nodes/nodeFuncs.h"
#include "parser/parsetree.h"
#include "pgstat.h"
@@ -452,6 +453,7 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_CASE_TESTVAL,
&&CASE_EEOP_MAKE_READONLY,
&&CASE_EEOP_IOCOERCE,
+ &&CASE_EEOP_IOCOERCE_SAFE,
&&CASE_EEOP_DISTINCT,
&&CASE_EEOP_NOT_DISTINCT,
&&CASE_EEOP_NULLIF,
@@ -1205,6 +1207,12 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_IOCOERCE_SAFE)
+ {
+ ExecEvalCoerceViaIOSafe(state, op);
+ EEO_NEXT();
+ }
+
EEO_CASE(EEOP_DISTINCT)
{
/*
@@ -2510,6 +2518,68 @@ ExecEvalParamExtern(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
errmsg("no value found for parameter %d", paramId)));
}
+/*
+ * Evaluate a CoerceViaIO node in soft-error mode.
+ *
+ * The source value is in op's result variable.
+ */
+void
+ExecEvalCoerceViaIOSafe(ExprState *state, ExprEvalStep *op)
+{
+ char *str;
+
+ /* call output function (similar to OutputFunctionCall) */
+ if (*op->resnull)
+ {
+ /* output functions are not called on nulls */
+ str = NULL;
+ }
+ else
+ {
+ FunctionCallInfo fcinfo_out;
+
+ fcinfo_out = op->d.iocoerce.fcinfo_data_out;
+ fcinfo_out->args[0].value = *op->resvalue;
+ fcinfo_out->args[0].isnull = false;
+
+ fcinfo_out->isnull = false;
+ str = DatumGetCString(FunctionCallInvoke(fcinfo_out));
+
+ /* OutputFunctionCall assumes result isn't null */
+ Assert(!fcinfo_out->isnull);
+ }
+
+ /* call input function (similar to InputFunctionCall) */
+ if (!op->d.iocoerce.finfo_in->fn_strict || str != NULL)
+ {
+ FunctionCallInfo fcinfo_in;
+
+ fcinfo_in = op->d.iocoerce.fcinfo_data_in;
+ fcinfo_in->args[0].value = PointerGetDatum(str);
+ fcinfo_in->args[0].isnull = *op->resnull;
+ /* second and third arguments are already set up */
+
+ /* ErrorSaveContext must be present. */
+ Assert(IsA(fcinfo_in->context, ErrorSaveContext));
+
+ fcinfo_in->isnull = false;
+ *op->resvalue = FunctionCallInvoke(fcinfo_in);
+
+ if (SOFT_ERROR_OCCURRED(fcinfo_in->context))
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ /* Should get null result if and only if str is NULL */
+ if (str == NULL)
+ Assert(*op->resnull);
+ else
+ Assert(!*op->resnull);
+ }
+}
+
/*
* Evaluate a SQLValueFunction expression.
*/
@@ -3730,7 +3800,7 @@ void
ExecEvalConstraintNotNull(ExprState *state, ExprEvalStep *op)
{
if (*op->resnull)
- ereport(ERROR,
+ errsave((Node *) op->d.domaincheck.escontext,
(errcode(ERRCODE_NOT_NULL_VIOLATION),
errmsg("domain %s does not allow null values",
format_type_be(op->d.domaincheck.resulttype)),
@@ -3745,7 +3815,7 @@ ExecEvalConstraintCheck(ExprState *state, ExprEvalStep *op)
{
if (!*op->d.domaincheck.checknull &&
!DatumGetBool(*op->d.domaincheck.checkvalue))
- ereport(ERROR,
+ errsave((Node *) op->d.domaincheck.escontext,
(errcode(ERRCODE_CHECK_VIOLATION),
errmsg("value for domain %s violates check constraint \"%s\"",
format_type_be(op->d.domaincheck.resulttype),
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index a3a0876bff..81856a9dc7 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1431,6 +1431,12 @@ llvm_compile_expr(ExprState *state)
break;
}
+ case EEOP_IOCOERCE_SAFE:
+ build_EvalXFunc(b, mod, "ExecEvalCoerceViaIOSafe",
+ v_state, op);
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
case EEOP_DISTINCT:
case EEOP_NOT_DISTINCT:
{
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 791902ff1f..3a4be09e50 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -162,6 +162,7 @@ void *referenced_functions[] =
ExecEvalRow,
ExecEvalRowNotNull,
ExecEvalRowNull,
+ ExecEvalCoerceViaIOSafe,
ExecEvalSQLValueFunction,
ExecEvalScalarArrayOp,
ExecEvalHashedScalarArrayOp,
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index 048573c2bc..c4fd933154 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -16,6 +16,7 @@
#include "executor/nodeAgg.h"
#include "nodes/execnodes.h"
+#include "nodes/miscnodes.h"
/* forward references to avoid circularity */
struct ExprEvalStep;
@@ -168,6 +169,7 @@ typedef enum ExprEvalOp
/* evaluate assorted special-purpose expression types */
EEOP_IOCOERCE,
+ EEOP_IOCOERCE_SAFE,
EEOP_DISTINCT,
EEOP_NOT_DISTINCT,
EEOP_NULLIF,
@@ -547,6 +549,7 @@ typedef struct ExprEvalStep
bool *checknull;
/* OID of domain type */
Oid resulttype;
+ ErrorSaveContext *escontext;
} domaincheck;
/* for EEOP_CONVERT_ROWTYPE */
@@ -776,6 +779,7 @@ extern void ExecEvalParamExec(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalParamExtern(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
+extern void ExecEvalCoerceViaIOSafe(ExprState *state, ExprEvalStep *op);
extern void ExecEvalSQLValueFunction(ExprState *state, ExprEvalStep *op);
extern void ExecEvalCurrentOfExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalNextValueExpr(ExprState *state, ExprEvalStep *op);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 5d7f17dee0..6a7118d300 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -34,6 +34,7 @@
#include "fmgr.h"
#include "lib/ilist.h"
#include "lib/pairingheap.h"
+#include "nodes/miscnodes.h"
#include "nodes/params.h"
#include "nodes/plannodes.h"
#include "nodes/tidbitmap.h"
@@ -129,6 +130,12 @@ typedef struct ExprState
Datum *innermost_domainval;
bool *innermost_domainnull;
+
+ /*
+ * For expression nodes that support soft errors. Should be set to NULL
+ * before calling ExecInitExprRec() if the caller wants errors thrown.
+ */
+ ErrorSaveContext *escontext;
} ExprState;
--
2.35.3
[application/octet-stream] v27-0004-JSON_TABLE.patch (166.9K, ../../CA+HiwqHgvd6akb4FxC7=GpmzXDAkGODXOBM+Q2g-UQ6+NK9sxw@mail.gmail.com/4-v27-0004-JSON_TABLE.patch)
download | inline diff:
From a4390a70ffd511f03d0f47abbd48d98da3d109a6 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Wed, 22 Nov 2023 13:19:05 +0900
Subject: [PATCH v27 4/4] JSON_TABLE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This feature allows jsonb data to be treated as a table and thus
used in a FROM clause like other tabular data. Data can be selected
from the jsonb using jsonpath expressions, and hoisted out of nested
structures in the jsonb to form multiple rows, more or less like an
outer join.
This also adds the PLAN clauses for JSON_TABLE, which allow the user
to specify how data from nested paths are joined, allowing
considerable freedom in shaping the tabular output of JSON_TABLE.
PLAN DEFAULT allows the user to specify the global strategies when
dealing with sibling or child nested paths. The is often sufficient
to achieve the necessary goal, and is considerably simpler than the
full PLAN clause, which allows the user to specify the strategy to be
used for each named nested path.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 496 ++++++++
src/backend/catalog/sql_features.txt | 6 +-
src/backend/commands/explain.c | 8 +-
src/backend/executor/execExprInterp.c | 6 +
src/backend/executor/nodeTableFuncscan.c | 27 +-
src/backend/nodes/makefuncs.c | 19 +
src/backend/nodes/nodeFuncs.c | 32 +
src/backend/parser/Makefile | 1 +
src/backend/parser/gram.y | 297 ++++-
src/backend/parser/meson.build | 1 +
src/backend/parser/parse_clause.c | 14 +-
src/backend/parser/parse_expr.c | 13 +
src/backend/parser/parse_jsontable.c | 769 ++++++++++++
src/backend/parser/parse_relation.c | 5 +-
src/backend/parser/parse_target.c | 3 +
src/backend/utils/adt/jsonpath_exec.c | 543 +++++++++
src/backend/utils/adt/ruleutils.c | 279 ++++-
src/include/nodes/execnodes.h | 2 +
src/include/nodes/makefuncs.h | 2 +
src/include/nodes/parsenodes.h | 95 ++
src/include/nodes/primnodes.h | 59 +-
src/include/parser/kwlist.h | 4 +
src/include/parser/parse_clause.h | 3 +
src/include/utils/jsonpath.h | 3 +
src/test/regress/expected/json_sqljson.out | 6 +
src/test/regress/expected/jsonb_sqljson.out | 1182 +++++++++++++++++++
src/test/regress/sql/json_sqljson.sql | 4 +
src/test/regress/sql/jsonb_sqljson.sql | 674 +++++++++++
src/tools/pgindent/typedefs.list | 13 +
29 files changed, 4537 insertions(+), 29 deletions(-)
create mode 100644 src/backend/parser/parse_jsontable.c
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 7962a8a1a4..f8fa6817cd 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17200,6 +17200,502 @@ array w/o UK? | t
</table>
</sect2>
+ <sect2 id="functions-sqljson-table">
+ <title>JSON_TABLE</title>
+ <indexterm>
+ <primary>json_table</primary>
+ </indexterm>
+
+ <para>
+ <function>json_table</function> is an SQL/JSON function which
+ queries <acronym>JSON</acronym> data
+ and presents the results as a relational view, which can be accessed as a
+ regular SQL table. You can only use <function>json_table</function> inside the
+ <literal>FROM</literal> clause of a <literal>SELECT</literal> statement.
+ </para>
+
+ <para>
+ Taking JSON data as input, <function>json_table</function> uses
+ a path expression to extract a part of the provided data that
+ will be used as a <firstterm>row pattern</firstterm> for the
+ constructed view. Each SQL/JSON item at the top level of the row pattern serves
+ as the source for a separate row in the constructed relational view.
+ </para>
+
+ <para>
+ To split the row pattern into columns, <function>json_table</function>
+ provides the <literal>COLUMNS</literal> clause that defines the
+ schema of the created view. For each column to be constructed,
+ this clause provides a separate path expression that evaluates
+ the row pattern, extracts a JSON item, and returns it as a
+ separate SQL value for the specified column. If the required value
+ is stored in a nested level of the row pattern, it can be extracted
+ using the <literal>NESTED PATH</literal> subclause. Joining the
+ columns returned by <literal>NESTED PATH</literal> can add multiple
+ new rows to the constructed view. Such rows are called
+ <firstterm>child rows</firstterm>, as opposed to the <firstterm>parent row</firstterm>
+ that generates them.
+ </para>
+
+ <para>
+ The rows produced by <function>JSON_TABLE</function> are laterally
+ joined to the row that generated them, so you do not have to explicitly join
+ the constructed view with the original table holding <acronym>JSON</acronym>
+ data. Optionally, you can specify how to join the columns returned
+ by <literal>NESTED PATH</literal> using the <literal>PLAN</literal> clause.
+ </para>
+
+ <para>
+ Each <literal>NESTED PATH</literal> clause can generate one or more
+ columns. Columns produced by <literal>NESTED PATH</literal>s at the
+ same level are considered to be <firstterm>siblings</firstterm>,
+ while a column produced by a <literal>NESTED PATH</literal> is
+ considered to be a child of the column produced by a
+ <literal>NESTED PATH</literal> or row expression at a higher level.
+ Sibling columns are always joined first. Once they are processed,
+ the resulting rows are joined to the parent row.
+ </para>
+
+ <para>
+ The syntax is:
+ </para>
+
+<synopsis>
+JSON_TABLE (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> AS <replaceable>json_path_name</replaceable> </optional> <optional> PASSING { <replaceable>value</replaceable> AS <replaceable>varname</replaceable> } <optional>, ...</optional> </optional>
+ COLUMNS ( <replaceable class="parameter">json_table_column</replaceable> <optional>, ...</optional> )
+ <optional> { <literal>ERROR</literal> | <literal>EMPTY</literal> } <literal>ON ERROR</literal> </optional>
+ <optional>
+ PLAN ( <replaceable class="parameter">json_table_plan</replaceable> ) |
+ PLAN DEFAULT ( { INNER | OUTER } <optional> , { CROSS | UNION } </optional>
+ | { CROSS | UNION } <optional> , { INNER | OUTER } </optional> )
+ </optional>
+)
+
+<phrase>
+where <replaceable class="parameter">json_table_column</replaceable> is:
+</phrase>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> FORMAT <replaceable>json_representation</replaceable>
+ <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> EXISTS <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { ERROR | TRUE | FALSE | UNKNOWN } ON ERROR </optional>
+ | NESTED PATH <replaceable>json_path_specification</replaceable> <optional> AS <replaceable>path_name</replaceable> </optional>
+ COLUMNS ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ | <replaceable>name</replaceable> FOR ORDINALITY
+<phrase>
+<replaceable>json_table_plan</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> <optional> { OUTER | INNER } <replaceable>json_table_plan_primary</replaceable> </optional>
+ | <replaceable>json_table_plan_primary</replaceable> { UNION <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+ | <replaceable>json_table_plan_primary</replaceable> { CROSS <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+<phrase>
+<replaceable>json_table_plan_primary</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> | ( <replaceable>json_table_plan</replaceable> )
+</synopsis>
+
+ <para>
+ Each syntax element is described below in more detail.
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional></literal>
+ </term>
+ <listitem>
+ <para>
+ The input data to query, the JSON path expression defining the query,
+ and an optional <literal>PASSING</literal> clause, which can provide data
+ values to the <replaceable>path_expression</replaceable>.
+ The result of the input data
+ evaluation is called the <firstterm>row pattern</firstterm>. The row
+ pattern is used as the source for row values in the constructed view.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>COLUMNS</literal>( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ The <literal>COLUMNS</literal> clause defining the schema of the
+ constructed view. In this clause, you must specify all the columns
+ to be filled with SQL/JSON items.
+ The <replaceable>json_table_column</replaceable>
+ expression has the following syntax variants:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>name</replaceable> <replaceable>type</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional></literal>
+ </term>
+ <listitem>
+
+ <para>
+ Inserts a single SQL/JSON item into each row of
+ the specified column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON EMPTY</literal> and
+ <literal>ON ERROR</literal> clauses to define how to handle missing values
+ or structural errors.
+ <literal>WRAPPER</literal> and <literal>QUOTES</literal> clauses can only
+ be used with JSON, array, and composite types.
+ These clauses have the same syntax and semantics as for
+ <function>json_value</function> and <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <literal>FORMAT</literal> <replaceable>json_representation</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a composite SQL/JSON
+ item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>WRAPPER</literal>, <literal>QUOTES</literal>,
+ <literal>ON EMPTY</literal> and <literal>ON ERROR</literal> clauses
+ to define additional settings for the returned SQL/JSON items.
+ These clauses have the same syntax and semantics as
+ for <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable>
+ <literal>EXISTS</literal> <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a boolean item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated,
+ a check whether any SQL/JSON items were returned is done, and
+ the column is filled with the resulting boolean value, one for each row.
+ The specified <parameter>type</parameter> should have a cast from the
+ <type>boolean</type>.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON ERROR</literal> clause to define
+ error behavior. This clause has the same syntax and semantics as
+ for <function>json_exists</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>NESTED PATH</literal> <replaceable>json_path_specification</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional>
+ <literal>COLUMNS</literal> ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ Extracts SQL/JSON items from nested levels of the row pattern,
+ generates one or more columns as defined by the <literal>COLUMNS</literal>
+ subclause, and inserts the extracted SQL/JSON items into each row of these columns.
+ The <replaceable>json_table_column</replaceable> expression in the
+ <literal>COLUMNS</literal> subclause uses the same syntax as in the
+ parent <literal>COLUMNS</literal> clause.
+ </para>
+
+ <para>
+ The <literal>NESTED PATH</literal> syntax is recursive,
+ so you can go down multiple nested levels by specifying several
+ <literal>NESTED PATH</literal> subclauses within each other.
+ It allows to unnest the hierarchy of JSON objects and arrays
+ in a single function invocation rather than chaining several
+ <function>JSON_TABLE</function> expressions in an SQL statement.
+ </para>
+
+ <para>
+ You can use the <literal>PLAN</literal> clause to define how
+ to join the columns returned by <literal>NESTED PATH</literal> clauses.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <literal>FOR ORDINALITY</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Adds an ordinality column that provides sequential row numbering.
+ You can have only one ordinality column per table. Row numbering
+ is 1-based. For child rows that result from the <literal>NESTED PATH</literal>
+ clauses, the parent row number is repeated.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>AS</literal> <replaceable>json_path_name</replaceable>
+ </term>
+ <listitem>
+
+ <para>
+ The optional <replaceable>json_path_name</replaceable> serves as an
+ identifier of the provided <replaceable>json_path_specification</replaceable>.
+ The path name must be unique and distinct from the column names.
+ When using the <literal>PLAN</literal> clause, you must specify the names
+ for all the paths, including the row pattern. Each path name can appear in
+ the <literal>PLAN</literal> clause only once.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN</literal> ( <replaceable>json_table_plan</replaceable> )
+ </term>
+ <listitem>
+
+ <para>
+ Defines how to join the data returned by <literal>NESTED PATH</literal>
+ clauses to the constructed view.
+ </para>
+ <para>
+ To join columns with parent/child relationship, you can use:
+ </para>
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>INNER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>INNER JOIN</literal>, so that the parent row
+ is omitted from the output if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>OUTER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>LEFT OUTER JOIN</literal>, so that the parent row
+ is always included into the output even if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>, with NULL values
+ inserted into the child columns if the corresponding
+ values are missing.
+ </para>
+ <para>
+ This is the default option for joining columns with parent/child relationship.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>
+ To join sibling columns, you can use:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>UNION</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each value produced by each of the sibling
+ columns. The columns from the other siblings are set to null.
+ </para>
+ <para>
+ This is the default option for joining sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>CROSS</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each combination of values from the sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN DEFAULT</literal> ( <literal><replaceable>OUTER | INNER</replaceable> <optional>, <replaceable>UNION | CROSS</replaceable> </optional></literal> )
+ </term>
+ <listitem>
+ <para>
+ The terms can also be specified in reverse order. The
+ <literal>INNER</literal> or <literal>OUTER</literal> option defines the
+ joining plan for parent/child columns, while <literal>UNION</literal> or
+ <literal>CROSS</literal> affects joins of sibling columns. This form
+ of <literal>PLAN</literal> overrides the default plan for
+ all columns at once. Even though the path names are not included in the
+ <literal>PLAN DEFAULT</literal> form, to conform to the SQL/JSON standard
+ they must be provided for all the paths if the <literal>PLAN</literal>
+ clause is used.
+ </para>
+ <para>
+ <literal>PLAN DEFAULT</literal> is simpler than specifying a complete
+ <literal>PLAN</literal>, and is often all that is required to get the desired
+ output.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>Examples</para>
+
+ <para>
+ In these examples the following small table storing some JSON data will be used:
+<programlisting>
+CREATE TABLE my_films ( js jsonb );
+
+INSERT INTO my_films VALUES (
+'{ "favorites" : [
+ { "kind" : "comedy", "films" : [
+ { "title" : "Bananas",
+ "director" : "Woody Allen"},
+ { "title" : "The Dinner Game",
+ "director" : "Francis Veber" } ] },
+ { "kind" : "horror", "films" : [
+ { "title" : "Psycho",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "thriller", "films" : [
+ { "title" : "Vertigo",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "drama", "films" : [
+ { "title" : "Yojimbo",
+ "director" : "Akira Kurosawa" } ] }
+ ] }');
+</programlisting>
+ </para>
+ <para>
+ Query the <structname>my_films</structname> table holding
+ some JSON data about the films and create a view that
+ distributes the film genre, title, and director between separate columns:
+<screen>
+SELECT jt.* FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites[*]' COLUMNS (
+ id FOR ORDINALITY,
+ kind text PATH '$.kind',
+ NESTED PATH '$.films[*]' COLUMNS (
+ title text PATH '$.title',
+ director text PATH '$.director'))) AS jt;
+----+----------+------------------+-------------------
+ id | kind | title | director
+----+----------+------------------+-------------------
+ 1 | comedy | Bananas | Woody Allen
+ 1 | comedy | The Dinner Game | Francis Veber
+ 2 | horror | Psycho | Alfred Hitchcock
+ 3 | thriller | Vertigo | Alfred Hitchcock
+ 4 | drama | Yojimbo | Akira Kurosawa
+ (5 rows)
+</screen>
+ </para>
+
+ <para>
+ Find a director that has done films in two different genres:
+<screen>
+SELECT
+ director1 AS director, title1, kind1, title2, kind2
+FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites' AS favs COLUMNS (
+ NESTED PATH '$[*]' AS films1 COLUMNS (
+ kind1 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film1 COLUMNS (
+ title1 text PATH '$.title',
+ director1 text PATH '$.director')
+ ),
+ NESTED PATH '$[*]' AS films2 COLUMNS (
+ kind2 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film2 COLUMNS (
+ title2 text PATH '$.title',
+ director2 text PATH '$.director'
+ )
+ )
+ )
+ PLAN (favs OUTER ((films1 INNER film1) CROSS (films2 INNER film2)))
+ ) AS jt
+ WHERE kind1 > kind2 AND director1 = director2;
+
+ director | title1 | kind1 | title2 | kind2
+------------------+---------+----------+--------+--------
+ Alfred Hitchcock | Vertigo | thriller | Psycho | horror
+(1 row)
+</screen>
+ </para>
+ </sect2>
+
<sect2 id="functions-sqljson-path">
<title>The SQL/JSON Path Language</title>
diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt
index 7598bd8f22..9500a80f4d 100644
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -551,10 +551,10 @@ T814 Colon in JSON_OBJECT or JSON_OBJECTAGG YES
T821 Basic SQL/JSON query operators YES
T822 SQL/JSON: IS JSON WITH UNIQUE KEYS predicate YES
T823 SQL/JSON: PASSING clause YES
-T824 JSON_TABLE: specific PLAN clause NO
+T824 JSON_TABLE: specific PLAN clause YES
T825 SQL/JSON: ON EMPTY and ON ERROR clauses YES
T826 General value expression in ON ERROR or ON EMPTY clauses YES
-T827 JSON_TABLE: sibling NESTED COLUMNS clauses NO
+T827 JSON_TABLE: sibling NESTED COLUMNS clauses YES
T828 JSON_QUERY YES
T829 JSON_QUERY: array wrapper options YES
T830 Enforcing unique keys in SQL/JSON constructor functions YES
@@ -565,7 +565,7 @@ T834 SQL/JSON path language: wildcard member accessor YES
T835 SQL/JSON path language: filter expressions YES
T836 SQL/JSON path language: starts with predicate YES
T837 SQL/JSON path language: regex_like predicate YES
-T838 JSON_TABLE: PLAN DEFAULT clause NO
+T838 JSON_TABLE: PLAN DEFAULT clause YES
T839 Formatted cast of datetimes to/from character strings NO
T840 Hex integer literals in SQL/JSON path language YES
T851 SQL/JSON: optional keywords for default syntax YES
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index f1d71bc54e..8e35525781 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -3892,7 +3892,13 @@ ExplainTargetRel(Plan *plan, Index rti, ExplainState *es)
break;
case T_TableFuncScan:
Assert(rte->rtekind == RTE_TABLEFUNC);
- objectname = "xmltable";
+ if (rte->tablefunc)
+ if (rte->tablefunc->functype == TFT_XMLTABLE)
+ objectname = "xmltable";
+ else /* Must be TFT_JSON_TABLE */
+ objectname = "json_table";
+ else
+ objectname = NULL;
objecttag = "Table Function Name";
break;
case T_ValuesScan:
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 697b9a84a9..8f9077d7dd 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -4345,6 +4345,12 @@ ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
break;
}
+ case JSON_TABLE_OP:
+ post_eval->jump_eval_coercion = -1;
+ *op->resvalue = item;
+ *op->resnull = false;
+ break;
+
default:
elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
return false;
diff --git a/src/backend/executor/nodeTableFuncscan.c b/src/backend/executor/nodeTableFuncscan.c
index a60dcd4943..0d7f518afd 100644
--- a/src/backend/executor/nodeTableFuncscan.c
+++ b/src/backend/executor/nodeTableFuncscan.c
@@ -28,6 +28,7 @@
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "utils/builtins.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/xml.h"
@@ -161,8 +162,9 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
scanstate->ss.ps.qual =
ExecInitQual(node->scan.plan.qual, &scanstate->ss.ps);
- /* Only XMLTABLE is supported currently */
- scanstate->routine = &XmlTableRoutine;
+ /* Only XMLTABLE and JSON_TABLE are supported currently */
+ scanstate->routine =
+ tf->functype == TFT_XMLTABLE ? &XmlTableRoutine : &JsonbTableRoutine;
scanstate->perTableCxt =
AllocSetContextCreate(CurrentMemoryContext,
@@ -182,6 +184,10 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
ExecInitExprList(tf->colexprs, (PlanState *) scanstate);
scanstate->coldefexprs =
ExecInitExprList(tf->coldefexprs, (PlanState *) scanstate);
+ scanstate->colvalexprs =
+ ExecInitExprList(tf->colvalexprs, (PlanState *) scanstate);
+ scanstate->passingvalexprs =
+ ExecInitExprList(tf->passingvalexprs, (PlanState *) scanstate);
scanstate->notnulls = tf->notnulls;
@@ -369,14 +375,17 @@ tfuncInitialize(TableFuncScanState *tstate, ExprContext *econtext, Datum doc)
routine->SetNamespace(tstate, ns_name, ns_uri);
}
- /* Install the row filter expression into the table builder context */
- value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
- if (isnull)
- ereport(ERROR,
- (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
- errmsg("row filter expression must not be null")));
+ if (routine->SetRowFilter)
+ {
+ /* Install the row filter expression into the table builder context */
+ value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
+ if (isnull)
+ ereport(ERROR,
+ (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
+ errmsg("row filter expression must not be null")));
- routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ }
/*
* Install the column filter expressions into the table builder context.
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index d5bb5cded6..dcfacb0beb 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -873,6 +873,25 @@ makeJsonBehavior(JsonBehaviorType type, Node *expr, int location)
return behavior;
}
+/*
+ * makeJsonTableJoinedPlan -
+ * creates a joined JsonTablePlan node
+ */
+Node *
+makeJsonTableJoinedPlan(JsonTablePlanJoinType type, Node *plan1, Node *plan2,
+ int location)
+{
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_JOINED;
+ n->join_type = type;
+ n->plan1 = castNode(JsonTablePlan, plan1);
+ n->plan2 = castNode(JsonTablePlan, plan2);
+ n->location = location;
+
+ return (Node *) n;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 62bf08ad27..f16bdd95a0 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -2697,6 +2697,10 @@ expression_tree_walker_impl(Node *node,
return true;
if (WALK(tf->coldefexprs))
return true;
+ if (WALK(tf->colvalexprs))
+ return true;
+ if (WALK(tf->passingvalexprs))
+ return true;
}
break;
default:
@@ -3757,6 +3761,8 @@ expression_tree_mutator_impl(Node *node,
MUTATE(newnode->rowexpr, tf->rowexpr, Node *);
MUTATE(newnode->colexprs, tf->colexprs, List *);
MUTATE(newnode->coldefexprs, tf->coldefexprs, List *);
+ MUTATE(newnode->colvalexprs, tf->colvalexprs, List *);
+ MUTATE(newnode->passingvalexprs, tf->passingvalexprs, List *);
return (Node *) newnode;
}
break;
@@ -4179,6 +4185,32 @@ raw_expression_tree_walker_impl(Node *node,
return true;
}
break;
+ case T_JsonTable:
+ {
+ JsonTable *jt = (JsonTable *) node;
+
+ if (WALK(jt->context_item))
+ return true;
+ if (WALK(jt->pathspec))
+ return true;
+ if (WALK(jt->passing))
+ return true;
+ if (WALK(jt->columns))
+ return true;
+ }
+ break;
+ case T_JsonTableColumn:
+ {
+ JsonTableColumn *jtc = (JsonTableColumn *) node;
+
+ if (WALK(jtc->typeName))
+ return true;
+ if (WALK(jtc->behavior))
+ return true;
+ if (jtc->coltype == JTC_NESTED && WALK(jtc->columns))
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/parser/Makefile b/src/backend/parser/Makefile
index 401c16686c..3162a01f30 100644
--- a/src/backend/parser/Makefile
+++ b/src/backend/parser/Makefile
@@ -23,6 +23,7 @@ OBJS = \
parse_enr.o \
parse_expr.o \
parse_func.o \
+ parse_jsontable.o \
parse_merge.o \
parse_node.o \
parse_oper.o \
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index e625b20b13..c15fcf2eb2 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -652,15 +652,32 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_aggregate_func
json_argument
json_behavior
+ json_table
+ json_table_column_definition
+ json_table_plan_clause_opt
+ json_table_plan
+ json_table_plan_simple
+ json_table_plan_parent_child
+ json_table_plan_outer
+ json_table_plan_inner
+ json_table_plan_sibling
+ json_table_plan_union
+ json_table_plan_cross
+ json_table_plan_primary
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
json_arguments
json_behavior_clause_opt
json_passing_clause_opt
+ json_table_column_definition_list
+%type <str> json_table_column_path_specification_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_quotes_clause_opt
+ json_table_default_plan_choices
+ json_table_default_plan_inner_outer
+ json_table_default_plan_union_cross
json_wrapper_behavior
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
@@ -730,7 +747,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
- JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_TABLE JSON_VALUE
KEEP KEY KEYS
@@ -741,8 +758,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
- NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
- NORMALIZE NORMALIZED
+ NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
+ NONE NORMALIZE NORMALIZED
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
@@ -750,8 +767,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PATH
+ PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -894,6 +911,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%left JOIN CROSS LEFT FULL RIGHT INNER_P NATURAL
+%nonassoc NESTED
+%left PATH
%%
/*
@@ -13420,6 +13439,21 @@ table_ref: relation_expr opt_alias_clause
$2->alias = $4;
$$ = (Node *) $2;
}
+ | json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $1);
+
+ jt->alias = $2;
+ $$ = (Node *) jt;
+ }
+ | LATERAL_P json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $2);
+
+ jt->alias = $3;
+ jt->lateral = true;
+ $$ = (Node *) jt;
+ }
;
@@ -13987,6 +14021,8 @@ xmltable_column_option_el:
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(true), @1); }
| NULL_P
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(false), @1); }
+ | PATH b_expr
+ { $$ = makeDefElem("path", $2, @1); }
;
xml_namespace_list:
@@ -16622,6 +16658,249 @@ json_quotes_clause_opt:
| /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
;
+json_table:
+ JSON_TABLE '('
+ json_value_expr ',' a_expr json_passing_clause_opt
+ COLUMNS '(' json_table_column_definition_list ')'
+ json_table_plan_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonTable *n = makeNode(JsonTable);
+
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->pathname = NULL;
+ n->passing = $6;
+ n->columns = $9;
+ n->plan = (JsonTablePlan *) $11;
+ n->behavior = $12;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_TABLE '('
+ json_value_expr ',' a_expr AS name json_passing_clause_opt
+ COLUMNS '(' json_table_column_definition_list ')'
+ json_table_plan_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonTable *n = makeNode(JsonTable);
+
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->pathname = $7;
+ n->passing = $8;
+ n->columns = $11;
+ n->plan = (JsonTablePlan *) $13;
+ n->behavior = $14;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_column_definition_list:
+ json_table_column_definition
+ { $$ = list_make1($1); }
+ | json_table_column_definition_list ',' json_table_column_definition
+ { $$ = lappend($1, $3); }
+ ;
+
+json_table_column_definition:
+ ColId FOR ORDINALITY
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FOR_ORDINALITY;
+ n->name = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_REGULAR;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->pathspec = $3;
+ n->wrapper = $4; /* disallowed during parse-analysis! */
+ n->quotes = $5; /* disallowed during parse-analysis! */
+ n->behavior = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename FORMAT_LA JSON json_encoding_clause_opt
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FORMATTED;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_JSON, $5, @1);
+ n->pathspec = $6;
+ n->wrapper = $7;
+ n->quotes = $8;
+ n->behavior = $9;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | ColId Typename
+ EXISTS json_table_column_path_specification_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_EXISTS;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = JSW_NONE;
+ n->quotes = JS_QUOTES_UNSPEC;
+ n->pathspec = $4;
+ n->behavior = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | NESTED path_opt Sconst
+ COLUMNS '(' json_table_column_definition_list ')'
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->columns = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | NESTED path_opt Sconst AS name
+ COLUMNS '(' json_table_column_definition_list ')'
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->columns = $8;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_column_path_specification_clause_opt:
+ PATH Sconst { $$ = $2; }
+ | /* EMPTY */ { $$ = NULL; }
+ ;
+
+path_opt:
+ PATH { }
+ | /* EMPTY */ { }
+ ;
+
+json_table_plan_clause_opt:
+ PLAN '(' json_table_plan ')' { $$ = $3; }
+ | PLAN DEFAULT '(' json_table_default_plan_choices ')'
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_DEFAULT;
+ n->join_type = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | /* EMPTY */ { $$ = NULL; }
+ ;
+
+json_table_plan:
+ json_table_plan_simple
+ | json_table_plan_parent_child
+ | json_table_plan_sibling
+ ;
+
+json_table_plan_simple:
+ name
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_SIMPLE;
+ n->pathname = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_plan_primary:
+ json_table_plan_simple { $$ = $1; }
+ | '(' json_table_plan ')'
+ {
+ castNode(JsonTablePlan, $2)->location = @1;
+ $$ = $2;
+ }
+ ;
+
+json_table_plan_outer:
+ json_table_plan_simple OUTER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_OUTER, $1, $3, @1); }
+ ;
+
+json_table_plan_inner:
+ json_table_plan_simple INNER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_INNER, $1, $3, @1); }
+ ;
+
+json_table_plan_parent_child:
+ json_table_plan_outer
+ | json_table_plan_inner
+ ;
+
+json_table_plan_union:
+ json_table_plan_primary UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ | json_table_plan_union UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ ;
+
+json_table_plan_cross:
+ json_table_plan_primary CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ | json_table_plan_cross CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ ;
+
+json_table_plan_sibling:
+ json_table_plan_union
+ | json_table_plan_cross
+ ;
+
+json_table_default_plan_choices:
+ json_table_default_plan_inner_outer { $$ = $1 | JSTPJ_UNION; }
+ | json_table_default_plan_union_cross { $$ = $1 | JSTPJ_OUTER; }
+ | json_table_default_plan_inner_outer ','
+ json_table_default_plan_union_cross { $$ = $1 | $3; }
+ | json_table_default_plan_union_cross ','
+ json_table_default_plan_inner_outer { $$ = $1 | $3; }
+ ;
+
+json_table_default_plan_inner_outer:
+ INNER_P { $$ = JSTPJ_INNER; }
+ | OUTER_P { $$ = JSTPJ_OUTER; }
+ ;
+
+json_table_default_plan_union_cross:
+ UNION { $$ = JSTPJ_UNION; }
+ | CROSS { $$ = JSTPJ_CROSS; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -17384,6 +17663,7 @@ unreserved_keyword:
| MOVE
| NAME_P
| NAMES
+ | NESTED
| NEW
| NEXT
| NFC
@@ -17418,6 +17698,8 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
+ | PLAN
| PLANS
| POLICY
| PRECEDING
@@ -17582,6 +17864,7 @@ col_name_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| LEAST
| NATIONAL
@@ -17950,6 +18233,7 @@ bare_label_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| KEEP
| KEY
@@ -17989,6 +18273,7 @@ bare_label_keyword:
| NATIONAL
| NATURAL
| NCHAR
+ | NESTED
| NEW
| NEXT
| NFC
@@ -18033,7 +18318,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
| PLACING
+ | PLAN
| PLANS
| POLICY
| POSITION
diff --git a/src/backend/parser/meson.build b/src/backend/parser/meson.build
index 94517a7f7d..0f0e4ac7ca 100644
--- a/src/backend/parser/meson.build
+++ b/src/backend/parser/meson.build
@@ -10,6 +10,7 @@ backend_sources += files(
'parse_enr.c',
'parse_expr.c',
'parse_func.c',
+ 'parse_jsontable.c',
'parse_merge.c',
'parse_node.c',
'parse_oper.c',
diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c
index 334b9b42bd..affd812619 100644
--- a/src/backend/parser/parse_clause.c
+++ b/src/backend/parser/parse_clause.c
@@ -697,7 +697,11 @@ transformRangeTableFunc(ParseState *pstate, RangeTableFunc *rtf)
char **names;
int colno;
- /* Currently only XMLTABLE is supported */
+ /*
+ * Currently we only support XMLTABLE here. See transformJsonTable() for
+ * JSON_TABLE support.
+ */
+ tf->functype = TFT_XMLTABLE;
constructName = "XMLTABLE";
docType = XMLOID;
@@ -1104,13 +1108,17 @@ transformFromClauseItem(ParseState *pstate, Node *n,
rtr->rtindex = nsitem->p_rtindex;
return (Node *) rtr;
}
- else if (IsA(n, RangeTableFunc))
+ else if (IsA(n, RangeTableFunc) || IsA(n, JsonTable))
{
/* table function is like a plain relation */
RangeTblRef *rtr;
ParseNamespaceItem *nsitem;
- nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ if (IsA(n, RangeTableFunc))
+ nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ else
+ nsitem = transformJsonTable(pstate, (JsonTable *) n);
+
*top_nsitem = nsitem;
*namespace = list_make1(nsitem);
rtr = makeNode(RangeTblRef);
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 4379008b4c..e587cd1ed5 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4358,6 +4358,19 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
JSON_BEHAVIOR_NULL,
jsexpr->returning);
break;
+
+ case JSON_TABLE_OP:
+ func_name = "JSON_TABLE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->returning = makeNode(JsonReturning);
+ jsexpr->returning->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ jsexpr->returning->typid = exprType(jsexpr->formatted_expr);
+ jsexpr->returning->typmod = -1;
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_EMPTY,
+ jsexpr->returning);
+ break;
}
return (Node *) jsexpr;
diff --git a/src/backend/parser/parse_jsontable.c b/src/backend/parser/parse_jsontable.c
new file mode 100644
index 0000000000..b3b16b824b
--- /dev/null
+++ b/src/backend/parser/parse_jsontable.c
@@ -0,0 +1,769 @@
+/*-------------------------------------------------------------------------
+ *
+ * parse_jsontable.c
+ * parsing of JSON_TABLE
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/parser/parse_jsontable.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "catalog/pg_collation.h"
+#include "catalog/pg_type.h"
+#include "miscadmin.h"
+#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
+#include "optimizer/optimizer.h"
+#include "parser/parse_clause.h"
+#include "parser/parse_collate.h"
+#include "parser/parse_expr.h"
+#include "parser/parse_relation.h"
+#include "parser/parse_type.h"
+#include "utils/builtins.h"
+#include "utils/json.h"
+#include "utils/lsyscache.h"
+
+/* Context for JSON_TABLE transformation */
+typedef struct JsonTableParseContext
+{
+ ParseState *pstate; /* parsing state */
+ JsonTable *table; /* untransformed node */
+ TableFunc *tablefunc; /* transformed node */
+ List *pathNames; /* list of all path and columns names */
+ int pathNameId; /* path name id counter */
+ Oid contextItemTypid; /* type oid of context item (json/jsonb) */
+} JsonTableParseContext;
+
+static JsonTableParent *transformJsonTableColumns(JsonTableParseContext * cxt,
+ JsonTablePlan *plan,
+ List *columns,
+ char *pathSpec,
+ char **pathName,
+ int location);
+
+static Node *
+makeStringConst(char *str, int location)
+{
+ A_Const *n = makeNode(A_Const);
+
+ n->val.node.type = T_String;
+ n->val.sval.sval = str;
+ n->location = location;
+
+ return (Node *) n;
+}
+
+/*
+ * Transform JSON_TABLE column
+ * - regular column into JSON_VALUE()
+ * - FORMAT JSON column into JSON_QUERY()
+ * - EXISTS column into JSON_EXISTS()
+ */
+static Node *
+transformJsonTableColumn(JsonTableColumn *jtc, Node *contextItemExpr,
+ List *passingArgs, bool errorOnError)
+{
+ JsonFuncExpr *jfexpr = makeNode(JsonFuncExpr);
+ JsonOutput *output = makeNode(JsonOutput);
+ char *pathspec;
+ JsonFormat *default_format;
+
+ jfexpr->op =
+ jtc->coltype == JTC_REGULAR ? JSON_VALUE_OP :
+ jtc->coltype == JTC_EXISTS ? JSON_EXISTS_OP : JSON_QUERY_OP;
+ jfexpr->output = output;
+ jfexpr->behavior = jtc->behavior;
+ if ((jfexpr->behavior == NIL ||
+ lsecond(jfexpr->behavior) == NULL) &&
+ errorOnError)
+ {
+ JsonBehavior *on_error = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL,
+ -1);
+
+ if (jfexpr->behavior == NIL)
+ jfexpr->behavior = list_make2(NULL, on_error);
+ else
+ {
+ jfexpr->behavior = list_delete_last(jfexpr->behavior);
+ jfexpr->behavior = lappend(jfexpr->behavior, on_error);
+ }
+ }
+ jfexpr->quotes = jtc->quotes;
+ jfexpr->wrapper = jtc->wrapper;
+ jfexpr->location = jtc->location;
+
+ output->typeName = jtc->typeName;
+ output->returning = makeNode(JsonReturning);
+ output->returning->format = jtc->format;
+
+ default_format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+
+ if (jtc->pathspec)
+ pathspec = jtc->pathspec;
+ else
+ {
+ /* Construct default path as '$."column_name"' */
+ StringInfoData path;
+
+ initStringInfo(&path);
+
+ appendStringInfoString(&path, "$.");
+ escape_json(&path, jtc->name);
+
+ pathspec = path.data;
+ }
+
+ jfexpr->context_item = makeJsonValueExpr((Expr *) contextItemExpr, NULL,
+ default_format);
+ jfexpr->pathspec = makeStringConst(pathspec, -1);
+ jfexpr->pathname = NULL;
+ jfexpr->passing = passingArgs;
+
+ return (Node *) jfexpr;
+}
+
+static bool
+isJsonTablePathNameDuplicate(JsonTableParseContext * cxt, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, cxt->pathNames)
+ {
+ if (!strcmp(pathname, (const char *) lfirst(lc)))
+ return true;
+ }
+
+ return false;
+}
+
+/* Register the column name in the path name list. */
+static void
+registerJsonTableColumn(JsonTableParseContext * cxt, char *colname)
+{
+ if (isJsonTablePathNameDuplicate(cxt, colname))
+ ereport(ERROR,
+ (errcode(ERRCODE_DUPLICATE_ALIAS),
+ errmsg("duplicate JSON_TABLE column name: %s", colname),
+ errhint("JSON_TABLE column names must be distinct from one another.")));
+
+ cxt->pathNames = lappend(cxt->pathNames, colname);
+}
+
+/* Recursively register all nested column names in the path name list. */
+static void
+registerAllJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ if (jtc->pathname)
+ registerJsonTableColumn(cxt, jtc->pathname);
+
+ registerAllJsonTableColumns(cxt, jtc->columns);
+ }
+ else
+ {
+ registerJsonTableColumn(cxt, jtc->name);
+ }
+ }
+}
+
+/* Generate a new unique JSON_TABLE path name. */
+static char *
+generateJsonTablePathName(JsonTableParseContext * cxt)
+{
+ char namebuf[32];
+ char *name = namebuf;
+
+ do
+ {
+ snprintf(namebuf, sizeof(namebuf), "json_table_path_%d",
+ ++cxt->pathNameId);
+ } while (isJsonTablePathNameDuplicate(cxt, name));
+
+ name = pstrdup(name);
+ cxt->pathNames = lappend(cxt->pathNames, name);
+
+ return name;
+}
+
+/* Collect sibling path names from plan to the specified list. */
+static void
+collectSiblingPathsInJsonTablePlan(JsonTablePlan *plan, List **paths)
+{
+ if (plan->plan_type == JSTP_SIMPLE)
+ *paths = lappend(*paths, plan->pathname);
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ *paths = lappend(*paths, plan->plan1->pathname);
+ }
+ else if (plan->join_type == JSTPJ_CROSS ||
+ plan->join_type == JSTPJ_UNION)
+ {
+ collectSiblingPathsInJsonTablePlan(plan->plan1, paths);
+ collectSiblingPathsInJsonTablePlan(plan->plan2, paths);
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE join type %d",
+ plan->join_type);
+ }
+}
+
+/*
+ * Validate child JSON_TABLE plan by checking that:
+ * - all nested columns have path names specified
+ * - all nested columns have corresponding node in the sibling plan
+ * - plan does not contain duplicate or extra nodes
+ */
+static void
+validateJsonTableChildPlan(ParseState *pstate, JsonTablePlan *plan,
+ List *columns)
+{
+ ListCell *lc1;
+ List *siblings = NIL;
+ int nchildren = 0;
+
+ if (plan)
+ collectSiblingPathsInJsonTablePlan(plan, &siblings);
+
+ foreach(lc1, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc1));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ ListCell *lc2;
+ bool found = false;
+
+ if (!jtc->pathname)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("nested JSON_TABLE columns must contain an explicit AS pathname specification if an explicit PLAN clause is used"),
+ parser_errposition(pstate, jtc->location)));
+
+ /* find nested path name in the list of sibling path names */
+ foreach(lc2, siblings)
+ {
+ if ((found = !strcmp(jtc->pathname, lfirst(lc2))))
+ break;
+ }
+
+ if (!found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node for nested path %s was not found in plan.", jtc->pathname),
+ parser_errposition(pstate, jtc->location)));
+
+ nchildren++;
+ }
+ }
+
+ if (list_length(siblings) > nchildren)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node contains some extra or duplicate sibling nodes."),
+ parser_errposition(pstate, plan ? plan->location : -1)));
+}
+
+static JsonTableColumn *
+findNestedJsonTableColumn(List *columns, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED &&
+ jtc->pathname &&
+ !strcmp(jtc->pathname, pathname))
+ return jtc;
+ }
+
+ return NULL;
+}
+
+static Node *
+transformNestedJsonTableColumn(JsonTableParseContext * cxt, JsonTableColumn *jtc,
+ JsonTablePlan *plan)
+{
+ JsonTableParent *node;
+ char *pathname = jtc->pathname;
+
+ node = transformJsonTableColumns(cxt, plan, jtc->columns, jtc->pathspec,
+ &pathname, jtc->location);
+
+ return (Node *) node;
+}
+
+static Node *
+makeJsonTableSiblingJoin(bool cross, Node *lnode, Node *rnode)
+{
+ JsonTableSibling *join = makeNode(JsonTableSibling);
+
+ join->larg = lnode;
+ join->rarg = rnode;
+ join->cross = cross;
+
+ return (Node *) join;
+}
+
+/*
+ * Recursively transform child JSON_TABLE plan.
+ *
+ * Default plan is transformed into a cross/union join of its nested columns.
+ * Simple and outer/inner plans are transformed into a JsonTableParent by
+ * finding and transforming corresponding nested column.
+ * Sibling plans are recursively transformed into a JsonTableSibling.
+ */
+static Node *
+transformJsonTableChildPlan(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns)
+{
+ JsonTableColumn *jtc = NULL;
+
+ if (!plan || plan->plan_type == JSTP_DEFAULT)
+ {
+ /* unspecified or default plan */
+ Node *res = NULL;
+ ListCell *lc;
+ bool cross = plan && (plan->join_type & JSTPJ_CROSS);
+
+ /* transform all nested columns into cross/union join */
+ foreach(lc, columns)
+ {
+ JsonTableColumn *col = castNode(JsonTableColumn, lfirst(lc));
+ Node *node;
+
+ if (col->coltype != JTC_NESTED)
+ continue;
+
+ node = transformNestedJsonTableColumn(cxt, col, plan);
+
+ /* join transformed node with previous sibling nodes */
+ res = res ? makeJsonTableSiblingJoin(cross, res, node) : node;
+ }
+
+ return res;
+ }
+ else if (plan->plan_type == JSTP_SIMPLE)
+ {
+ jtc = findNestedJsonTableColumn(columns, plan->pathname);
+ }
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ jtc = findNestedJsonTableColumn(columns, plan->plan1->pathname);
+ }
+ else
+ {
+ Node *node1 = transformJsonTableChildPlan(cxt, plan->plan1,
+ columns);
+ Node *node2 = transformJsonTableChildPlan(cxt, plan->plan2,
+ columns);
+
+ return makeJsonTableSiblingJoin(plan->join_type == JSTPJ_CROSS,
+ node1, node2);
+ }
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE plan type %d", plan->plan_type);
+
+ if (!jtc)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name was %s not found in nested columns list.",
+ plan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ return transformNestedJsonTableColumn(cxt, jtc, plan);
+}
+
+/* Check whether type is json/jsonb, array, or record. */
+static bool
+typeIsComposite(Oid typid)
+{
+ char typtype;
+
+ if (typid == JSONOID ||
+ typid == JSONBOID ||
+ typid == RECORDOID ||
+ type_is_array(typid))
+ return true;
+
+ typtype = get_typtype(typid);
+
+ if (typtype == TYPTYPE_COMPOSITE)
+ return true;
+
+ if (typtype == TYPTYPE_DOMAIN)
+ return typeIsComposite(getBaseType(typid));
+
+ return false;
+}
+
+/* Append transformed non-nested JSON_TABLE columns to the TableFunc node */
+static void
+appendJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *col;
+ ParseState *pstate = cxt->pstate;
+ JsonTable *jt = cxt->table;
+ TableFunc *tf = cxt->tablefunc;
+ bool ordinality_found = false;
+ JsonBehavior *on_error = jt->behavior != NIL ? lsecond(jt->behavior) : NULL;
+ bool errorOnError = on_error &&
+ on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ foreach(col, columns)
+ {
+ JsonTableColumn *rawc = castNode(JsonTableColumn, lfirst(col));
+ Oid typid;
+ int32 typmod;
+ Node *colexpr;
+
+ if (rawc->name)
+ {
+ /* make sure column names are unique */
+ ListCell *colname;
+
+ foreach(colname, tf->colnames)
+ if (!strcmp((const char *) colname, rawc->name))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("column name \"%s\" is not unique",
+ rawc->name),
+ parser_errposition(pstate, rawc->location)));
+
+ tf->colnames = lappend(tf->colnames,
+ makeString(pstrdup(rawc->name)));
+ }
+
+ /*
+ * Determine the type and typmod for the new column. FOR ORDINALITY
+ * columns are INTEGER by standard; the others are user-specified.
+ */
+ switch (rawc->coltype)
+ {
+ case JTC_FOR_ORDINALITY:
+ if (ordinality_found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use more than one FOR ORDINALITY column"),
+ parser_errposition(pstate, rawc->location)));
+ ordinality_found = true;
+ colexpr = NULL;
+ typid = INT4OID;
+ typmod = -1;
+ break;
+
+ case JTC_REGULAR:
+ typenameTypeIdAndMod(pstate, rawc->typeName, &typid, &typmod);
+
+ /*
+ * Use implicit FORMAT JSON for composite types (arrays and
+ * records).
+ *
+ * For scalar columns, require an explicit FORMAT JSON clause
+ * to get the specified WRAPPER and QUOTES behavior.
+ */
+ if (typeIsComposite(typid))
+ rawc->coltype = JTC_FORMATTED;
+ else if (rawc->wrapper != JSW_NONE)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use WITH WRAPPER clause for scalar columns"
+ " without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+ else if (rawc->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use OMIT/KEEP QUOTES clause for scalar columns"
+ " without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+
+ /* FALLTHROUGH */
+ case JTC_EXISTS:
+ case JTC_FORMATTED:
+ {
+ Node *je;
+ CaseTestExpr *param = makeNode(CaseTestExpr);
+
+ param->collation = InvalidOid;
+ param->typeId = cxt->contextItemTypid;
+ param->typeMod = -1;
+
+ if (rawc->wrapper != JSW_NONE &&
+ rawc->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use WITH WRAPPER clause for formatted colunmns"
+ " without also specifying OMIT/KEEP QUOTES"),
+ parser_errposition(pstate, rawc->location)));
+
+ je = transformJsonTableColumn(rawc, (Node *) param,
+ NIL, errorOnError);
+
+ colexpr = transformExpr(pstate, je, EXPR_KIND_FROM_FUNCTION);
+ assign_expr_collations(pstate, colexpr);
+
+ typid = exprType(colexpr);
+ typmod = exprTypmod(colexpr);
+ break;
+ }
+
+ case JTC_NESTED:
+ continue;
+
+ default:
+ elog(ERROR, "unknown JSON_TABLE column type: %d", rawc->coltype);
+ break;
+ }
+
+ tf->coltypes = lappend_oid(tf->coltypes, typid);
+ tf->coltypmods = lappend_int(tf->coltypmods, typmod);
+ tf->colcollations = lappend_oid(tf->colcollations, get_typcollation(typid));
+ tf->colvalexprs = lappend(tf->colvalexprs, colexpr);
+ }
+}
+
+/*
+ * Create transformed JSON_TABLE parent plan node by appending all non-nested
+ * columns to the TableFunc node and remembering their indices in the
+ * colvalexprs list.
+ */
+static JsonTableParent *
+makeParentJsonTableNode(JsonTableParseContext * cxt, char *pathSpec, char *pathName,
+ List *columns)
+{
+ JsonTableParent *node = makeNode(JsonTableParent);
+ JsonBehavior *on_error = cxt->table->behavior != NIL ?
+ lsecond(cxt->table->behavior) : NULL;
+
+ node->path = makeNode(JsonTablePath);
+ node->path->value = makeConst(JSONPATHOID, -1, InvalidOid, -1,
+ DirectFunctionCall1(jsonpath_in,
+ CStringGetDatum(pathSpec)),
+ false, false);
+ if (pathName)
+ node->path->name = pstrdup(pathName);
+
+ /* save start of column range */
+ node->colMin = list_length(cxt->tablefunc->colvalexprs);
+
+ appendJsonTableColumns(cxt, columns);
+
+ /* save end of column range */
+ node->colMax = list_length(cxt->tablefunc->colvalexprs) - 1;
+
+ node->errorOnError = on_error && on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ return node;
+}
+
+static JsonTableParent *
+transformJsonTableColumns(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns, char *pathSpec, char **pathName,
+ int location)
+{
+ JsonTableParent *node;
+ JsonTablePlan *childPlan;
+ bool defaultPlan = !plan || plan->plan_type == JSTP_DEFAULT;
+
+ if (!*pathName)
+ {
+ if (cxt->table->plan)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE expression"),
+ errdetail("JSON_TABLE columns must contain "
+ "explicit AS pathname specification if "
+ "explicit PLAN clause is used"),
+ parser_errposition(cxt->pstate, location)));
+
+ *pathName = generateJsonTablePathName(cxt);
+ }
+
+ if (defaultPlan)
+ childPlan = plan;
+ else
+ {
+ /* validate parent and child plans */
+ JsonTablePlan *parentPlan;
+
+ if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type != JSTPJ_INNER &&
+ plan->join_type != JSTPJ_OUTER)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Expected INNER or OUTER JSON_TABLE plan node."),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ parentPlan = plan->plan1;
+ childPlan = plan->plan2;
+
+ Assert(parentPlan->plan_type != JSTP_JOINED);
+ Assert(parentPlan->pathname);
+ }
+ else
+ {
+ parentPlan = plan;
+ childPlan = NULL;
+ }
+
+ if (strcmp(parentPlan->pathname, *pathName))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name mismatch: expected %s but %s is given.",
+ *pathName, parentPlan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ validateJsonTableChildPlan(cxt->pstate, childPlan, columns);
+ }
+
+ /* transform only non-nested columns */
+ node = makeParentJsonTableNode(cxt, pathSpec, *pathName, columns);
+
+ if (childPlan || defaultPlan)
+ {
+ /* transform recursively nested columns */
+ node->child = transformJsonTableChildPlan(cxt, childPlan, columns);
+ if (node->child)
+ node->outerJoin = !plan || (plan->join_type & JSTPJ_OUTER);
+ /* else: default plan case, no children found */
+ }
+
+ return node;
+}
+
+/*
+ * transformJsonTable -
+ * Transform a raw JsonTable into TableFunc.
+ *
+ * Transform the document-generating expression, the row-generating expression,
+ * the column-generating expressions, and the default value expressions.
+ */
+ParseNamespaceItem *
+transformJsonTable(ParseState *pstate, JsonTable *jt)
+{
+ JsonTableParseContext cxt;
+ TableFunc *tf = makeNode(TableFunc);
+ JsonFuncExpr *jfe = makeNode(JsonFuncExpr);
+ JsonExpr *je;
+ JsonTablePlan *plan = jt->plan;
+ char *rootPathName = jt->pathname;
+ char *rootPath;
+ bool is_lateral;
+
+ cxt.pstate = pstate;
+ cxt.table = jt;
+ cxt.tablefunc = tf;
+ cxt.pathNames = NIL;
+ cxt.pathNameId = 0;
+
+ if (rootPathName)
+ registerJsonTableColumn(&cxt, rootPathName);
+
+ registerAllJsonTableColumns(&cxt, jt->columns);
+
+#if 0 /* XXX it' unclear from the standard whether
+ * root path name is mandatory or not */
+ if (plan && plan->plan_type != JSTP_DEFAULT && !rootPathName)
+ {
+ /* Assign root path name and create corresponding plan node */
+ JsonTablePlan *rootNode = makeNode(JsonTablePlan);
+ JsonTablePlan *rootPlan = (JsonTablePlan *)
+ makeJsonTableJoinedPlan(JSTPJ_OUTER, (Node *) rootNode,
+ (Node *) plan, jt->location);
+
+ rootPathName = generateJsonTablePathName(&cxt);
+
+ rootNode->plan_type = JSTP_SIMPLE;
+ rootNode->pathname = rootPathName;
+
+ plan = rootPlan;
+ }
+#endif
+
+ jfe->op = JSON_TABLE_OP;
+ jfe->context_item = jt->context_item;
+ jfe->pathspec = jt->pathspec;
+ jfe->pathname = jt->pathname;
+ jfe->passing = jt->passing;
+ jfe->behavior = jt->behavior;
+ jfe->location = jt->location;
+
+ /*
+ * We make lateral_only names of this level visible, whether or not the
+ * RangeTableFunc is explicitly marked LATERAL. This is needed for SQL
+ * spec compliance and seems useful on convenience grounds for all
+ * functions in FROM.
+ *
+ * (LATERAL can't nest within a single pstate level, so we don't need
+ * save/restore logic here.)
+ */
+ Assert(!pstate->p_lateral_active);
+ pstate->p_lateral_active = true;
+
+ tf->functype = TFT_JSON_TABLE;
+ tf->docexpr = transformExpr(pstate, (Node *) jfe, EXPR_KIND_FROM_FUNCTION);
+
+ cxt.contextItemTypid = exprType(tf->docexpr);
+
+ if (!IsA(jt->pathspec, A_Const) ||
+ castNode(A_Const, jt->pathspec)->val.node.type != T_String)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("only string constants supported in JSON_TABLE path specification"),
+ parser_errposition(pstate,
+ exprLocation(jt->pathspec))));
+
+ rootPath = castNode(A_Const, jt->pathspec)->val.sval.sval;
+
+ tf->plan = (Node *) transformJsonTableColumns(&cxt, plan, jt->columns,
+ rootPath, &rootPathName,
+ exprLocation(jt->pathspec));
+
+ /* Also save a copy of the PASSING arguments in the TableFunc node. */
+ je = (JsonExpr *) tf->docexpr;
+ tf->passingvalexprs = copyObject(je->passing_values);
+
+ tf->ordinalitycol = -1; /* undefine ordinality column number */
+ tf->location = jt->location;
+
+ pstate->p_lateral_active = false;
+
+ /*
+ * Mark the RTE as LATERAL if the user said LATERAL explicitly, or if
+ * there are any lateral cross-references in it.
+ */
+ is_lateral = jt->lateral || contain_vars_of_level((Node *) tf, 0);
+
+ return addRangeTableEntryForTableFunc(pstate,
+ tf, jt->alias, is_lateral, true);
+}
diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c
index 864ea9b0d5..3ac7e113f0 100644
--- a/src/backend/parser/parse_relation.c
+++ b/src/backend/parser/parse_relation.c
@@ -2073,7 +2073,8 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
Assert(list_length(tf->coltypmods) == list_length(tf->colnames));
Assert(list_length(tf->colcollations) == list_length(tf->colnames));
- refname = alias ? alias->aliasname : pstrdup("xmltable");
+ refname = alias ? alias->aliasname :
+ pstrdup(tf->functype == TFT_XMLTABLE ? "xmltable" : "json_table");
rte->rtekind = RTE_TABLEFUNC;
rte->relid = InvalidOid;
@@ -2096,7 +2097,7 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
errmsg("%s function has %d columns available but %d columns specified",
- "XMLTABLE",
+ tf->functype == TFT_XMLTABLE ? "XMLTABLE" : "JSON_TABLE",
list_length(tf->colnames), numaliases)));
rte->eref = eref;
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 8f81624f97..ca7228ea41 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -2002,6 +2002,9 @@ FigureColnameInternal(Node *node, char **name)
case JSON_VALUE_OP:
*name = "json_value";
return 2;
+ case JSON_TABLE_OP:
+ *name = "json_table";
+ return 2;
}
break;
default:
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 6c5602c64d..bb559d033f 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -61,10 +61,12 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
+#include "executor/execExpr.h"
#include "funcapi.h"
#include "lib/stringinfo.h"
#include "miscadmin.h"
#include "nodes/miscnodes.h"
+#include "nodes/nodeFuncs.h"
#include "regex/regex.h"
#include "utils/builtins.h"
#include "utils/date.h"
@@ -75,6 +77,8 @@
#include "utils/guc.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
#include "utils/timestamp.h"
#include "utils/varlena.h"
@@ -156,6 +160,61 @@ typedef struct JsonValueListIterator
ListCell *next;
} JsonValueListIterator;
+/* Structures for JSON_TABLE execution */
+
+typedef enum JsonTablePlanStateType
+{
+ JSON_TABLE_SCAN_STATE = 0,
+ JSON_TABLE_JOIN_STATE,
+} JsonTablePlanStateType;
+
+typedef struct JsonTablePlanState
+{
+ JsonTablePlanStateType type;
+
+ struct JsonTablePlanState *parent;
+ struct JsonTablePlanState *nested;
+} JsonTablePlanState;
+
+typedef struct JsonTableScanState
+{
+ JsonTablePlanState plan;
+
+ MemoryContext mcxt;
+ JsonPath *path;
+ List *args;
+ JsonValueList found;
+ JsonValueListIterator iter;
+ Datum current;
+ int ordinal;
+ bool currentIsNull;
+ bool outerJoin;
+ bool errorOnError;
+ bool advanceNested;
+ bool reset;
+} JsonTableScanState;
+
+typedef struct JsonTableJoinState
+{
+ JsonTablePlanState plan;
+
+ JsonTablePlanState *left;
+ JsonTablePlanState *right;
+ bool cross;
+ bool advanceRight;
+} JsonTableJoinState;
+
+/* random number to identify JsonTableExecContext */
+#define JSON_TABLE_EXEC_CONTEXT_MAGIC 418352867
+
+typedef struct JsonTableExecContext
+{
+ int magic;
+ JsonTableScanState **colexprscans;
+ JsonTableScanState *root;
+ bool empty;
+} JsonTableExecContext;
+
/* strict/lax flags is decomposed into four [un]wrap/error flags */
#define jspStrictAbsenseOfErrors(cxt) (!(cxt)->laxMode)
#define jspAutoUnwrap(cxt) ((cxt)->laxMode)
@@ -248,6 +307,7 @@ static JsonPathExecResult getArrayIndex(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonbValue *jb, int32 *index);
static JsonBaseObjectInfo setBaseObject(JsonPathExecContext *cxt,
JsonbValue *jbv, int32 id);
+static void JsonValueListClear(JsonValueList *jvl);
static void JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv);
static int JsonValueListLength(const JsonValueList *jvl);
static bool JsonValueListIsEmpty(JsonValueList *jvl);
@@ -265,6 +325,14 @@ static JsonbValue *wrapItemsInArray(const JsonValueList *items);
static int compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
bool useTz, bool *cast_error);
+
+static JsonTablePlanState * JsonTableInitPlanState(JsonTableExecContext * cxt,
+ Node *plan,
+ JsonTablePlanState * parent);
+static bool JsonTablePlanNextRow(JsonTablePlanState * state);
+static bool JsonTableScanNextRow(JsonTableScanState *scan);
+
+
/****************** User interface to JsonPath executor ********************/
/*
@@ -2518,6 +2586,13 @@ setBaseObject(JsonPathExecContext *cxt, JsonbValue *jbv, int32 id)
return baseObject;
}
+static void
+JsonValueListClear(JsonValueList *jvl)
+{
+ jvl->singleton = NULL;
+ jvl->list = NULL;
+}
+
static void
JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv)
{
@@ -3123,3 +3198,471 @@ JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
}
}
+
+/************************ JSON_TABLE functions ***************************/
+
+/*
+ * Returns private data from executor state. Ensure validity by check with
+ * MAGIC number.
+ */
+static inline JsonTableExecContext *
+GetJsonTableExecContext(TableFuncScanState *state, const char *fname)
+{
+ JsonTableExecContext *result;
+
+ if (!IsA(state, TableFuncScanState))
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+ result = (JsonTableExecContext *) state->opaque;
+ if (result->magic != JSON_TABLE_EXEC_CONTEXT_MAGIC)
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+
+ return result;
+}
+
+/* Recursively initialize JSON_TABLE scan / join state */
+static JsonTableJoinState *
+JsonTableInitJoinState(JsonTableExecContext * cxt, JsonTableSibling *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTableJoinState *join = palloc0(sizeof(*join));
+
+ join->plan.type = JSON_TABLE_JOIN_STATE;
+ /* parent and nested not set. */
+
+ join->cross = plan->cross;
+ join->left = JsonTableInitPlanState(cxt, plan->larg, parent);
+ join->right = JsonTableInitPlanState(cxt, plan->rarg, parent);
+
+ return join;
+}
+
+static JsonTableScanState *
+JsonTableInitScanState(JsonTableExecContext * cxt, JsonTableParent *plan,
+ JsonTablePlanState * parent,
+ List *args, MemoryContext mcxt)
+{
+ JsonTableScanState *scan = palloc0(sizeof(*scan));
+ int i;
+
+ scan->plan.type = JSON_TABLE_SCAN_STATE;
+ scan->plan.parent = parent;
+
+ scan->outerJoin = plan->outerJoin;
+ scan->errorOnError = plan->errorOnError;
+ scan->path = DatumGetJsonPathP(plan->path->value->constvalue);
+ scan->args = args;
+ scan->mcxt = AllocSetContextCreate(mcxt, "JsonTableExecContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Set after settings scan->args and scan->mcxt, because the recursive
+ * call wants to use those values.
+ */
+ scan->plan.nested = plan->child ?
+ JsonTableInitPlanState(cxt, plan->child, (JsonTablePlanState *) scan) :
+ NULL;
+
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+
+ for (i = plan->colMin; i <= plan->colMax; i++)
+ cxt->colexprscans[i] = scan;
+
+ return scan;
+}
+
+/* Recursively initialize JSON_TABLE scan state */
+static JsonTablePlanState *
+JsonTableInitPlanState(JsonTableExecContext * cxt, Node *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTablePlanState *state;
+
+ if (IsA(plan, JsonTableSibling))
+ {
+ JsonTableSibling *join = castNode(JsonTableSibling, plan);
+
+ state = (JsonTablePlanState *)
+ JsonTableInitJoinState(cxt, join, parent);
+ }
+ else
+ {
+ JsonTableParent *scan = castNode(JsonTableParent, plan);
+ JsonTableScanState *parent_scan = (JsonTableScanState *) parent;
+
+ Assert(parent_scan);
+ state = (JsonTablePlanState *)
+ JsonTableInitScanState(cxt, scan, parent, parent_scan->args,
+ parent_scan->mcxt);
+ }
+
+ return state;
+}
+
+/*
+ * JsonTableInitOpaque
+ * Fill in TableFuncScanState->opaque for JsonTable processor
+ */
+static void
+JsonTableInitOpaque(TableFuncScanState *state, int natts)
+{
+ JsonTableExecContext *cxt;
+ PlanState *ps = &state->ss.ps;
+ TableFuncScan *tfs = castNode(TableFuncScan, ps->plan);
+ TableFunc *tf = tfs->tablefunc;
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+ JsonExpr *je = castNode(JsonExpr, tf->docexpr);
+ List *args = NIL;
+
+ cxt = palloc0(sizeof(JsonTableExecContext));
+ cxt->magic = JSON_TABLE_EXEC_CONTEXT_MAGIC;
+
+ if (state->passingvalexprs)
+ {
+ ListCell *exprlc;
+ ListCell *namelc;
+
+ Assert(list_length(state->passingvalexprs) ==
+ list_length(je->passing_names));
+ forboth(exprlc, state->passingvalexprs,
+ namelc, je->passing_names)
+ {
+ ExprState *state = lfirst_node(ExprState, exprlc);
+ String *name = lfirst_node(String, namelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = pstrdup(name->sval);
+ var->typid = exprType((Node *) state->expr);
+ var->typmod = exprTypmod((Node *) state->expr);
+
+ /*
+ * Evaluate the expression and save the value to be returned by
+ * GetJsonPathVar().
+ */
+ var->value = ExecEvalExpr(state, ps->ps_ExprContext,
+ &var->isnull);
+
+ args = lappend(args, var);
+ }
+ }
+
+ cxt->colexprscans = palloc(sizeof(JsonTableScanState *) *
+ list_length(tf->colvalexprs));
+
+ cxt->root = JsonTableInitScanState(cxt, root, NULL, args,
+ CurrentMemoryContext);
+ state->opaque = cxt;
+}
+
+/* Reset scan iterator to the beginning of the item list */
+static void
+JsonTableRescan(JsonTableScanState *scan)
+{
+ JsonValueListInitIterator(&scan->found, &scan->iter);
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ scan->advanceNested = false;
+ scan->ordinal = 0;
+}
+
+/* Reset context item of a scan, execute JSON path and reset a scan */
+static void
+JsonTableResetContextItem(JsonTableScanState *scan, Datum item)
+{
+ MemoryContext oldcxt;
+ JsonPathExecResult res;
+ Jsonb *js = (Jsonb *) DatumGetJsonbP(item);
+
+ JsonValueListClear(&scan->found);
+
+ MemoryContextResetOnly(scan->mcxt);
+
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+
+ res = executeJsonPath(scan->path, scan->args, GetJsonPathVar, js,
+ scan->errorOnError, &scan->found,
+ false /* FIXME */ );
+
+ MemoryContextSwitchTo(oldcxt);
+
+ if (jperIsError(res))
+ {
+ Assert(!scan->errorOnError);
+ JsonValueListClear(&scan->found); /* EMPTY ON ERROR case */
+ }
+
+ JsonTableRescan(scan);
+}
+
+/*
+ * JsonTableSetDocument
+ * Install the input document
+ */
+static void
+JsonTableSetDocument(TableFuncScanState *state, Datum value)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableSetDocument");
+
+ JsonTableResetContextItem(cxt->root, value);
+}
+
+/* Recursively reset scan and its child nodes */
+static void
+JsonTableRescanRecursive(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTableRescanRecursive(join->left);
+ JsonTableRescanRecursive(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan = (JsonTableScanState *) state;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ JsonTableRescan(scan);
+ if (scan->plan.nested)
+ JsonTableRescanRecursive(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a cross/union joined scan.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTablePlanNextRow(JsonTablePlanState * state)
+{
+ JsonTableJoinState *join;
+
+ if (state->type == JSON_TABLE_SCAN_STATE)
+ return JsonTableScanNextRow((JsonTableScanState *) state);
+
+ join = (JsonTableJoinState *) state;
+ if (join->advanceRight)
+ {
+ /* fetch next inner row */
+ if (JsonTablePlanNextRow(join->right))
+ return true;
+
+ /* inner rows are exhausted */
+ if (join->cross)
+ join->advanceRight = false; /* next outer row */
+ else
+ return false; /* end of scan */
+ }
+
+ while (!join->advanceRight)
+ {
+ /* fetch next outer row */
+ bool left = JsonTablePlanNextRow(join->left);
+
+ if (join->cross)
+ {
+ if (!left)
+ return false; /* end of scan */
+
+ JsonTableRescanRecursive(join->right);
+
+ if (!JsonTablePlanNextRow(join->right))
+ continue; /* next outer row */
+
+ join->advanceRight = true; /* next inner row */
+ }
+ else if (!left)
+ {
+ if (!JsonTablePlanNextRow(join->right))
+ return false; /* end of scan */
+
+ join->advanceRight = true; /* next inner row */
+ }
+
+ break;
+ }
+
+ return true;
+}
+
+/* Recursively set 'reset' flag of scan and its child nodes */
+static void
+JsonTablePlanReset(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTablePlanReset(join->left);
+ JsonTablePlanReset(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ scan = (JsonTableScanState *) state;
+ scan->reset = true;
+ scan->advanceNested = false;
+
+ if (scan->plan.nested)
+ JsonTablePlanReset(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a simple scan with outer/inner joined nested subscans.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTableScanNextRow(JsonTableScanState *scan)
+{
+ /* reset context item if requested */
+ if (scan->reset)
+ {
+ JsonTableScanState *parent_scan =
+ (JsonTableScanState *) scan->plan.parent;
+
+ Assert(parent_scan && !parent_scan->currentIsNull);
+ JsonTableResetContextItem(scan, parent_scan->current);
+ scan->reset = false;
+ }
+
+ if (scan->advanceNested)
+ {
+ /* fetch next nested row */
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested)
+ return true;
+ }
+
+ for (;;)
+ {
+ /* fetch next row */
+ JsonbValue *jbv = JsonValueListNext(&scan->found, &scan->iter);
+ MemoryContext oldcxt;
+
+ if (!jbv)
+ {
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ return false; /* end of scan */
+ }
+
+ /* set current row item */
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+ scan->current = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ scan->currentIsNull = false;
+ MemoryContextSwitchTo(oldcxt);
+
+ scan->ordinal++;
+
+ if (!scan->plan.nested)
+ break;
+
+ JsonTablePlanReset(scan->plan.nested);
+
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested || scan->outerJoin)
+ break;
+ }
+
+ return true;
+}
+
+/*
+ * JsonTableFetchRow
+ * Prepare the next "current" tuple for upcoming GetValue calls.
+ * Returns FALSE if the row-filter expression returned no more rows.
+ */
+static bool
+JsonTableFetchRow(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableFetchRow");
+
+ if (cxt->empty)
+ return false;
+
+ return JsonTableScanNextRow(cxt->root);
+}
+
+/*
+ * JsonTableGetValue
+ * Return the value for column number 'colnum' for the current row.
+ *
+ * This leaks memory, so be sure to reset often the context in which it's
+ * called.
+ */
+static Datum
+JsonTableGetValue(TableFuncScanState *state, int colnum,
+ Oid typid, int32 typmod, bool *isnull)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableGetValue");
+ ExprContext *econtext = state->ss.ps.ps_ExprContext;
+ ExprState *estate = list_nth(state->colvalexprs, colnum);
+ JsonTableScanState *scan = cxt->colexprscans[colnum];
+ Datum result;
+
+ if (scan->currentIsNull) /* NULL from outer/union join */
+ {
+ result = (Datum) 0;
+ *isnull = true;
+ }
+ else if (estate) /* regular column */
+ {
+ Datum saved_caseValue = econtext->caseValue_datum;
+ bool saved_caseIsNull = econtext->caseValue_isNull;
+
+ /* Pass the value for CaseTestExpr that may be present in colexpr */
+ econtext->caseValue_datum = scan->current;
+ econtext->caseValue_isNull = false;
+
+ result = ExecEvalExpr(estate, econtext, isnull);
+
+ econtext->caseValue_datum = saved_caseValue;
+ econtext->caseValue_isNull = saved_caseIsNull;
+ }
+ else
+ {
+ result = Int32GetDatum(scan->ordinal); /* ordinality column */
+ *isnull = false;
+ }
+
+ return result;
+}
+
+/*
+ * JsonTableDestroyOpaque
+ */
+static void
+JsonTableDestroyOpaque(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableDestroyOpaque");
+
+ /* not valid anymore */
+ cxt->magic = 0;
+
+ state->opaque = NULL;
+}
+
+const TableFuncRoutine JsonbTableRoutine =
+{
+ JsonTableInitOpaque,
+ JsonTableSetDocument,
+ NULL,
+ NULL,
+ NULL,
+ JsonTableFetchRow,
+ JsonTableGetValue,
+ JsonTableDestroyOpaque
+};
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index a1745c89ff..46f49c7365 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -522,6 +522,8 @@ static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
static void get_json_path_spec(Node *path_spec, deparse_context *context,
bool showimplicit);
+static void get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8627,7 +8629,8 @@ get_json_behavior(JsonBehavior *behavior, deparse_context *context,
/*
* get_json_expr_options
*
- * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS and
+ * JSON_TABLE columns.
*/
static void
get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
@@ -9874,6 +9877,9 @@ get_rule_expr(Node *node, deparse_context *context,
case JSON_VALUE_OP:
appendStringInfoString(buf, "JSON_VALUE(");
break;
+ default:
+ elog(ERROR, "unexpected JsonExpr type: %d", jexpr->op);
+ break;
}
get_rule_expr(jexpr->formatted_expr, context, showimplicit);
@@ -11240,16 +11246,14 @@ get_sublink_expr(SubLink *sublink, deparse_context *context)
/* ----------
- * get_tablefunc - Parse back a table function
+ * get_xmltable - Parse back a XMLTABLE function
* ----------
*/
static void
-get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+get_xmltable(TableFunc *tf, deparse_context *context, bool showimplicit)
{
StringInfo buf = context->buf;
- /* XMLTABLE is the only existing implementation. */
-
appendStringInfoString(buf, "XMLTABLE(");
if (tf->ns_uris != NIL)
@@ -11340,6 +11344,271 @@ get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
appendStringInfoChar(buf, ')');
}
+/*
+ * get_json_nested_columns - Parse back nested JSON_TABLE columns
+ */
+static void
+get_json_table_nested_columns(TableFunc *tf, Node *node,
+ deparse_context *context, bool showimplicit,
+ bool needcomma)
+{
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_nested_columns(tf, n->larg, context, showimplicit,
+ needcomma);
+ get_json_table_nested_columns(tf, n->rarg, context, showimplicit, true);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ if (needcomma)
+ appendStringInfoChar(context->buf, ',');
+
+ appendStringInfoChar(context->buf, ' ');
+ appendContextKeyword(context, "NESTED PATH ", 0, 0, 0);
+ get_const_expr(n->path->value, context, -1);
+ appendStringInfo(context->buf, " AS %s", quote_identifier(n->path->name));
+ get_json_table_columns(tf, n, context, showimplicit);
+ }
+}
+
+/*
+ * get_json_table_plan - Parse back a JSON_TABLE plan
+ */
+static void
+get_json_table_plan(TableFunc *tf, Node *node, deparse_context *context,
+ bool parenthesize)
+{
+ if (parenthesize)
+ appendStringInfoChar(context->buf, '(');
+
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_plan(tf, n->larg, context,
+ IsA(n->larg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->larg)->child);
+
+ appendStringInfoString(context->buf, n->cross ? " CROSS " : " UNION ");
+
+ get_json_table_plan(tf, n->rarg, context,
+ IsA(n->rarg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->rarg)->child);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ appendStringInfoString(context->buf, quote_identifier(n->path->name));
+
+ if (n->child)
+ {
+ appendStringInfoString(context->buf,
+ n->outerJoin ? " OUTER " : " INNER ");
+ get_json_table_plan(tf, n->child, context,
+ IsA(n->child, JsonTableSibling));
+ }
+ }
+
+ if (parenthesize)
+ appendStringInfoChar(context->buf, ')');
+}
+
+/*
+ * get_json_table_columns - Parse back JSON_TABLE columns
+ */
+static void
+get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ ListCell *lc_colname;
+ ListCell *lc_coltype;
+ ListCell *lc_coltypmod;
+ ListCell *lc_colvalexpr;
+ int colnum = 0;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "COLUMNS (", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forfour(lc_colname, tf->colnames,
+ lc_coltype, tf->coltypes,
+ lc_coltypmod, tf->coltypmods,
+ lc_colvalexpr, tf->colvalexprs)
+ {
+ char *colname = strVal(lfirst(lc_colname));
+ JsonExpr *colexpr;
+ Oid typid;
+ int32 typmod;
+ bool ordinality;
+ JsonBehaviorType default_behavior;
+
+ typid = lfirst_oid(lc_coltype);
+ typmod = lfirst_int(lc_coltypmod);
+ colexpr = castNode(JsonExpr, lfirst(lc_colvalexpr));
+
+ if (colnum < node->colMin)
+ {
+ colnum++;
+ continue;
+ }
+
+ if (colnum > node->colMax)
+ break;
+
+ if (colnum > node->colMin)
+ appendStringInfoString(buf, ", ");
+
+ colnum++;
+
+ ordinality = !colexpr;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ appendStringInfo(buf, "%s %s", quote_identifier(colname),
+ ordinality ? "FOR ORDINALITY" :
+ format_type_with_typemod(typid, typmod));
+ if (ordinality)
+ continue;
+
+ if (colexpr->op == JSON_EXISTS_OP)
+ {
+ appendStringInfoString(buf, " EXISTS");
+ default_behavior = JSON_BEHAVIOR_FALSE;
+ }
+ else
+ {
+ if (colexpr->op == JSON_QUERY_OP)
+ {
+ char typcategory;
+ bool typispreferred;
+
+ get_type_category_preferred(typid, &typcategory, &typispreferred);
+
+ if (typcategory == TYPCATEGORY_STRING)
+ appendStringInfoString(buf,
+ colexpr->format->format_type == JS_FORMAT_JSONB ?
+ " FORMAT JSONB" : " FORMAT JSON");
+ }
+
+ default_behavior = JSON_BEHAVIOR_NULL;
+ }
+
+ if (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR)
+ default_behavior = JSON_BEHAVIOR_ERROR;
+
+ appendStringInfoString(buf, " PATH ");
+
+ get_json_path_spec(colexpr->path_spec, context, showimplicit);
+
+ get_json_expr_options(colexpr, context, default_behavior);
+ }
+
+ if (node->child)
+ get_json_table_nested_columns(tf, node->child, context, showimplicit,
+ node->colMax >= node->colMin);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_json_table - Parse back a JSON_TABLE function
+ * ----------
+ */
+static void
+get_json_table(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+
+ appendStringInfoString(buf, "JSON_TABLE(");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_const_expr(root->path->value, context, -1);
+
+ appendStringInfo(buf, " AS %s", quote_identifier(root->path->name));
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PASSING ", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr((Node *) lfirst(lc2), context, false);
+ appendStringInfo(buf, " AS %s",
+ quote_identifier((lfirst_node(String, lc1))->sval)
+ );
+ }
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+ }
+
+ get_json_table_columns(tf, root, context, showimplicit);
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PLAN ", 0, 0, 0);
+ get_json_table_plan(tf, (Node *) root, context, true);
+
+ if (jexpr->on_error->btype != JSON_BEHAVIOR_EMPTY)
+ get_json_behavior(jexpr->on_error, context, "ERROR");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_tablefunc - Parse back a table function
+ * ----------
+ */
+static void
+get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ /* XMLTABLE and JSON_TABLE are the only existing implementations. */
+
+ if (tf->functype == TFT_XMLTABLE)
+ get_xmltable(tf, context, showimplicit);
+ else if (tf->functype == TFT_JSON_TABLE)
+ get_json_table(tf, context, showimplicit);
+}
+
/* ----------
* get_from_clause - Parse back a FROM clause
*
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 6a7118d300..2fa0328977 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1882,6 +1882,8 @@ typedef struct TableFuncScanState
ExprState *rowexpr; /* state for row-generating expression */
List *colexprs; /* state for column-generating expression */
List *coldefexprs; /* state for column default expressions */
+ List *colvalexprs; /* state for column value expression */
+ List *passingvalexprs; /* state for PASSING argument expression */
List *ns_names; /* same as TableFunc.ns_names */
List *ns_uris; /* list of states of namespace URI exprs */
Bitmapset *notnulls; /* nullability flag for each output column */
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index a850a1928b..a0b864deda 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -113,6 +113,8 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
+extern Node *makeJsonTableJoinedPlan(JsonTablePlanJoinType type,
+ Node *plan1, Node *plan2, int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index f0560c3737..a0bcca36d8 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1709,6 +1709,19 @@ typedef enum JsonQuotes
*/
typedef char *JsonPathSpec;
+/*
+ * JsonTableColumnType -
+ * enumeration of JSON_TABLE column types
+ */
+typedef enum JsonTableColumnType
+{
+ JTC_FOR_ORDINALITY,
+ JTC_REGULAR,
+ JTC_EXISTS,
+ JTC_FORMATTED,
+ JTC_NESTED,
+} JsonTableColumnType;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1741,6 +1754,7 @@ typedef struct JsonFuncExpr
JsonExprOp op; /* expression type */
JsonValueExpr *context_item; /* context item expression */
Node *pathspec; /* JSON path specification expression */
+ char *pathname; /* path name, if any */
List *passing; /* list of PASSING clause arguments, if any */
JsonOutput *output; /* output clause, if specified */
List *behavior; /* ON EMPTY / ERROR behaviors, list of two
@@ -1750,6 +1764,87 @@ typedef struct JsonFuncExpr
int location; /* token location, or -1 if unknown */
} JsonFuncExpr;
+/*
+ * JsonTableColumn -
+ * untransformed representation of JSON_TABLE column
+ */
+typedef struct JsonTableColumn
+{
+ NodeTag type;
+ JsonTableColumnType coltype; /* column type */
+ char *name; /* column name */
+ TypeName *typeName; /* column type name */
+ char *pathspec; /* path specification, if any */
+ char *pathname; /* path name, if any */
+ JsonFormat *format; /* JSON format clause, if specified */
+ JsonWrapper wrapper; /* WRAPPER behavior for formatted columns */
+ JsonQuotes quotes; /* omit or keep quotes on scalar strings? */
+ List *columns; /* nested columns */
+ List *behavior; /* ON EMPTY / ERROR behaviors, list of two
+ * JsonBehavior nodes in that order */
+ int location; /* token location, or -1 if unknown */
+} JsonTableColumn;
+
+/*
+ * JsonTablePlanType -
+ * flags for JSON_TABLE plan node types representation
+ */
+typedef enum JsonTablePlanType
+{
+ JSTP_DEFAULT,
+ JSTP_SIMPLE,
+ JSTP_JOINED,
+} JsonTablePlanType;
+
+/*
+ * JsonTablePlanJoinType -
+ * flags for JSON_TABLE join types representation
+ */
+typedef enum JsonTablePlanJoinType
+{
+ JSTPJ_INNER = 0x01,
+ JSTPJ_OUTER = 0x02,
+ JSTPJ_CROSS = 0x04,
+ JSTPJ_UNION = 0x08,
+} JsonTablePlanJoinType;
+
+typedef struct JsonTablePlan JsonTablePlan;
+
+/*
+ * JsonTablePlan -
+ * untransformed representation of JSON_TABLE plan node
+ */
+struct JsonTablePlan
+{
+ NodeTag type;
+ JsonTablePlanType plan_type; /* plan type */
+ JsonTablePlanJoinType join_type; /* join type (for joined plan only) */
+ JsonTablePlan *plan1; /* first joined plan */
+ JsonTablePlan *plan2; /* second joined plan */
+ char *pathname; /* path name (for simple plan only) */
+ int location; /* token location, or -1 if unknown */
+};
+
+/*
+ * JsonTable -
+ * untransformed representation of JSON_TABLE
+ */
+typedef struct JsonTable
+{
+ NodeTag type;
+ JsonValueExpr *context_item; /* context item expression */
+ Node *pathspec; /* JSON path specification expression */
+ char *pathname; /* path name, if any */
+ List *passing; /* list of PASSING clause arguments, if any */
+ List *columns; /* list of JsonTableColumn */
+ JsonTablePlan *plan; /* join plan, if specified */
+ List *behavior; /* ON EMPTY / ERROR behaviors, list of two
+ * JsonBehavior nodes in that order */
+ Alias *alias; /* table alias in FROM clause */
+ bool lateral; /* does it have LATERAL prefix? */
+ int location; /* token location, or -1 if unknown */
+} JsonTable;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 7504b09ca8..75bbd14a35 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -94,8 +94,14 @@ typedef struct RangeVar
int location;
} RangeVar;
+typedef enum TableFuncType
+{
+ TFT_XMLTABLE,
+ TFT_JSON_TABLE,
+} TableFuncType;
+
/*
- * TableFunc - node for a table function, such as XMLTABLE.
+ * TableFunc - node for a table function, such as XMLTABLE or JSON_TABLE.
*
* Entries in the ns_names list are either String nodes containing
* literal namespace names, or NULL pointers to represent DEFAULT.
@@ -103,6 +109,8 @@ typedef struct RangeVar
typedef struct TableFunc
{
NodeTag type;
+ /* XMLTABLE or JSON_TABLE */
+ TableFuncType functype;
/* list of namespace URI expressions */
List *ns_uris pg_node_attr(query_jumble_ignore);
/* list of namespace names or NULL */
@@ -123,8 +131,14 @@ typedef struct TableFunc
List *colexprs;
/* list of column default expressions */
List *coldefexprs pg_node_attr(query_jumble_ignore);
+ /* list of column value expressions */
+ List *colvalexprs pg_node_attr(query_jumble_ignore);
+ /* list of PASSING argument expressions */
+ List *passingvalexprs pg_node_attr(query_jumble_ignore);
/* nullability flag for each output column */
Bitmapset *notnulls pg_node_attr(query_jumble_ignore);
+ /* JSON_TABLE plan */
+ Node *plan pg_node_attr(query_jumble_ignore);
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1561,6 +1575,7 @@ typedef enum JsonExprOp
JSON_VALUE_OP, /* JSON_VALUE() */
JSON_QUERY_OP, /* JSON_QUERY() */
JSON_EXISTS_OP, /* JSON_EXISTS() */
+ JSON_TABLE_OP, /* JSON_TABLE() */
} JsonExprOp;
/*
@@ -1800,6 +1815,48 @@ typedef struct JsonExpr
int location; /* token location, or -1 if unknown */
} JsonExpr;
+/*
+ * JsonTablePath
+ * A JSON path expression to be computed as part of evaluating
+ * a JSON_TABLE plan node
+ */
+typedef struct JsonTablePath
+{
+ NodeTag type;
+
+ Const *value;
+ char *name;
+} JsonTablePath;
+
+/*
+ * JsonTableParent -
+ * transformed representation of parent JSON_TABLE plan node
+ */
+typedef struct JsonTableParent
+{
+ NodeTag type;
+ JsonTablePath *path;
+ Node *child; /* nested columns, if any */
+ bool outerJoin; /* outer or inner join for nested columns? */
+ int colMin; /* min column index in the resulting column
+ * list */
+ int colMax; /* max column index in the resulting column
+ * list */
+ bool errorOnError; /* ERROR/EMPTY ON ERROR behavior */
+} JsonTableParent;
+
+/*
+ * JsonTableSibling -
+ * transformed representation of joined sibling JSON_TABLE plan node
+ */
+typedef struct JsonTableSibling
+{
+ NodeTag type;
+ Node *larg; /* left join node */
+ Node *rarg; /* right join node */
+ bool cross; /* cross or union join? */
+} JsonTableSibling;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 0954d9fc7b..f88a6c9ac6 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -242,6 +242,7 @@ PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_table", JSON_TABLE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -284,6 +285,7 @@ PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("nested", NESTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -334,7 +336,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_clause.h b/src/include/parser/parse_clause.h
index 8e26482db4..7152a8ef48 100644
--- a/src/include/parser/parse_clause.h
+++ b/src/include/parser/parse_clause.h
@@ -51,4 +51,7 @@ extern List *addTargetToSortList(ParseState *pstate, TargetEntry *tle,
extern Index assignSortGroupRef(TargetEntry *tle, List *tlist);
extern bool targetIsInSortList(TargetEntry *tle, Oid sortop, List *sortList);
+/* functions in parse_jsontable.c */
+extern ParseNamespaceItem *transformJsonTable(ParseState *pstate, JsonTable *jt);
+
#endif /* PARSE_CLAUSE_H */
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index 5a37133847..7eb0ccf4e4 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -15,6 +15,7 @@
#define JSONPATH_H
#include "fmgr.h"
+#include "executor/tablefunc.h"
#include "nodes/pg_list.h"
#include "nodes/primnodes.h"
#include "utils/jsonb.h"
@@ -288,4 +289,6 @@ extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
bool *error, List *vars);
+extern PGDLLIMPORT const TableFuncRoutine JsonbTableRoutine;
+
#endif
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
index 04d5cc74e3..5d0c6b6fdd 100644
--- a/src/test/regress/expected/json_sqljson.out
+++ b/src/test/regress/expected/json_sqljson.out
@@ -16,3 +16,9 @@ ERROR: JSON_QUERY() is not yet implemented for the json type
LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
^
HINT: Try casting the argument to jsonb
+-- JSON_TABLE
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
+ERROR: JSON_TABLE() is not yet implemented for the json type
+LINE 1: SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo ...
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
index cb727e930a..7e8ae6a696 100644
--- a/src/test/regress/expected/jsonb_sqljson.out
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -1030,3 +1030,1185 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.dateti
ERROR: functions in index expression must be marked IMMUTABLE
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+-- JSON_TABLE
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+ERROR: syntax error at or near "("
+LINE 1: SELECT JSON_TABLE('[]', '$');
+ ^
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ERROR: syntax error at or near ")"
+LINE 1: SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ ^
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+ERROR: JSON_TABLE function has 1 columns available but 2 columns specified
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+ foo
+-----
+(0 rows)
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+ item | foo
+------+-----
+ 123 |
+(1 row)
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | int | text | char(4) | bool | numeric | domain | js | jb
+---------------------------------------------------------------------------------------+----+-----+---------+---------+------+---------+---------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [] | | | | | | | | |
+ {} | 1 | | | | | | | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1 | 1.23 | 1.23 | | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | 2 | 2 | 2 | | 2 | 2 | "2" | "2"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | aaaaaaa | aaaa | | | aaaaaaa | "aaaaaaa" | "aaaaaaa"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | foo | foo | | | | "foo" | "foo"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | | | | | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | 0 | false | fals | f | | false | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | 1 | true | true | t | | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | | | | | | | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | [1,2] | [1,2 | | | [1,2] | "[1,2]" | "[1,2]"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | "str" | "str | | | "str" | "\"str\"" | "\"str\""
+(14 rows)
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+ js | id | jst | jsc | jsv | jsb | jsbq
+---------------------------------------------------------------------------------------+----+--------------+------+------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | 1 | 1
+ [] | | | | | |
+ {} | 1 | {} | {} | {} | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1.23 | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | "2" | "2" | "2" | "2" | 2
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | "aaaaaaa" | "aaa | "aaa | "aaaaaaa" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | "foo" | "foo | "foo | "foo" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | null | null | null | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | false | fals | fals | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | true | true | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | {"aaa": 123} | {"aa | {"aa | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | "[1,2]" | "[1, | "[1, | "[1,2]" | [1, 2]
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | "\"str\"" | "\"s | "\"s | "\"str\"" | "str"
+(14 rows)
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+ js | id | exists1 | exists2 | exists3 | exists4
+---------------------------------------------------------------------------------------+----+---------+---------+---------+---------
+ 1 | 1 | f | 0 | | false
+ [] | | | | |
+ {} | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | t | 1 | 1 | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | f | 0 | | false
+(14 rows)
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | aaa | aaa1 | js2 | jsb2w | jsb2q | ia | ta | jba
+---------------------------------------------------------------------------------------+----+-----+------+--------------+----------------+--------------+----+----+-----
+ 1 | 1 | | | 1 | [1] | 1 | | |
+ [] | | | | | | | | |
+ {} | 1 | | | {} | [{}] | {} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | | | 1 | [1] | 1 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | | | 1.23 | [1.23] | 1.23 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | | | "2" | ["2"] | 2 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | | "aaaaaaa" | ["aaaaaaa"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | | "foo" | ["foo"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | null | [null] | null | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | | | false | [false] | false | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | | | true | [true] | true | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | 123 | 123 | {"aaa": 123} | [{"aaa": 123}] | {"aaa": 123} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | | "[1,2]" | ["[1,2]"] | [1, 2] | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | | "\"str\"" | ["\"str\""] | "str" | | |
+(14 rows)
+
+-- JSON_TABLE: Test backward parsing
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+\sv jsonb_table_view
+CREATE OR REPLACE VIEW public.jsonb_table_view AS
+ SELECT id,
+ "int",
+ text,
+ "char(4)",
+ bool,
+ "numeric",
+ domain,
+ js,
+ jb,
+ jst,
+ jsc,
+ jsv,
+ jsb,
+ jsbq,
+ aaa,
+ aaa1,
+ exists1,
+ exists2,
+ exists3,
+ js2,
+ jsb2w,
+ jsb2q,
+ ia,
+ ta,
+ jba,
+ a1,
+ b1,
+ a11,
+ a21,
+ a22
+ FROM JSON_TABLE(
+ 'null'::jsonb, '$[*]' AS json_table_path_1
+ PASSING
+ 1 + 2 AS a,
+ '"foo"'::json AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" integer PATH '$',
+ text text PATH '$',
+ "char(4)" character(4) PATH '$',
+ bool boolean PATH '$',
+ "numeric" numeric PATH '$',
+ domain jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc character(4) FORMAT JSON PATH '$',
+ jsv character varying(4) FORMAT JSON PATH '$',
+ jsb jsonb PATH '$',
+ jsbq jsonb PATH '$' OMIT QUOTES,
+ aaa integer PATH '$."aaa"',
+ aaa1 integer PATH '$."aaa"',
+ exists1 boolean EXISTS PATH '$."aaa"',
+ exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia integer[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1
+ COLUMNS (
+ a1 integer PATH '$."a1"',
+ b1 text PATH '$."b1"',
+ NESTED PATH '$[*]' AS "p1 1"
+ COLUMNS (
+ a11 text PATH '$."a11"'
+ )
+ ),
+ NESTED PATH '$[2]' AS p2
+ COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1"
+ COLUMNS (
+ a21 text PATH '$."a21"'
+ ),
+ NESTED PATH '$[*]' AS p22
+ COLUMNS (
+ a22 text PATH '$."a22"'
+ )
+ )
+ )
+ PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22))))
+ )
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+ QUERY PLAN
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ Table Function Scan on "json_table"
+ Output: "json_table".id, "json_table"."int", "json_table".text, "json_table"."char(4)", "json_table".bool, "json_table"."numeric", "json_table".domain, "json_table".js, "json_table".jb, "json_table".jst, "json_table".jsc, "json_table".jsv, "json_table".jsb, "json_table".jsbq, "json_table".aaa, "json_table".aaa1, "json_table".exists1, "json_table".exists2, "json_table".exists3, "json_table".js2, "json_table".jsb2w, "json_table".jsb2q, "json_table".ia, "json_table".ta, "json_table".jba, "json_table".a1, "json_table".b1, "json_table".a11, "json_table".a21, "json_table".a22
+ Table Function Call: JSON_TABLE('null'::jsonb, '$[*]' AS json_table_path_1 PASSING 3 AS a, '"foo"'::jsonb AS "b c" COLUMNS (id FOR ORDINALITY, "int" integer PATH '$', text text PATH '$', "char(4)" character(4) PATH '$', bool boolean PATH '$', "numeric" numeric PATH '$', domain jsonb_test_domain PATH '$', js json PATH '$', jb jsonb PATH '$', jst text FORMAT JSON PATH '$', jsc character(4) FORMAT JSON PATH '$', jsv character varying(4) FORMAT JSON PATH '$', jsb jsonb PATH '$', jsbq jsonb PATH '$' OMIT QUOTES, aaa integer PATH '$."aaa"', aaa1 integer PATH '$."aaa"', exists1 boolean EXISTS PATH '$."aaa"', exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR, exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR, js2 json PATH '$', jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER, jsb2q jsonb PATH '$' OMIT QUOTES, ia integer[] PATH '$', ta text[] PATH '$', jba jsonb[] PATH '$', NESTED PATH '$[1]' AS p1 COLUMNS (a1 integer PATH '$."a1"', b1 text PATH '$."b1"', NESTED PATH '$[*]' AS "p1 1" COLUMNS (a11 text PATH '$."a11"')), NESTED PATH '$[2]' AS p2 COLUMNS ( NESTED PATH '$[*]' AS "p2:1" COLUMNS (a21 text PATH '$."a21"'), NESTED PATH '$[*]' AS p22 COLUMNS (a22 text PATH '$."a22"'))) PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22)))))
+(3 rows)
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+ERROR: cannot use more than one FOR ORDINALITY column
+LINE 1: ..._TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR OR...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+ id | a
+----+---
+ 1 | 1
+(1 row)
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+ js | a
+-------+---
+ 1 | 1
+ "err" |
+(2 rows)
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+ERROR: no SQL/JSON item
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: no SQL/JSON item
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 1
+(1 row)
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+ a
+---
+ 0
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to smallint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to bigint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to real
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 E...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+ a
+-----
+ fal
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to json
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to jsonb
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EX...
+ ^
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+ERROR: cannot use OMIT/KEEP QUOTES clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+ item
+---------
+ "world"
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+ERROR: cannot use WITH WRAPPER clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+ item
+-----------
+ ["world"]
+(1 row)
+
+-- JSON_TABLE: nested paths and plans
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb '[]', '$' -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 4: NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: b
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+-- JSON_TABLE: plan validation
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p1)
+ ^
+DETAIL: Path name mismatch: expected p0 but p1 is given.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 UNION p1 UNION p11)
+ ^
+DETAIL: Expected INNER or OUTER JSON_TABLE plan node.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 8: NESTED PATH '$' AS p2 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p2 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 5: NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ ^
+DETAIL: Plan node for nested path p11 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+ ^
+DETAIL: Plan node contains some extra or duplicate sibling nodes.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 6: NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ^
+DETAIL: Plan node for nested path p12 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 9: NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ ^
+DETAIL: Plan node for nested path p21 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+ bar | foo | baz
+-----+-----+-----
+(0 rows)
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb 'null', 'strict $[*]' -- without root path name
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- JSON_TABLE: plan execution
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+ n | a | c | b
+---+----+----+---
+ 1 | 1 | |
+ 2 | 2 | 10 |
+ 2 | 2 | |
+ 2 | 2 | 20 |
+ 2 | 2 | | 1
+ 2 | 2 | | 2
+ 2 | 2 | | 3
+ 3 | 3 | | 1
+ 3 | 3 | | 2
+ 4 | -1 | | 1
+ 4 | -1 | | 2
+(11 rows)
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+ n | a | b | b1 | c | c1 | b
+---+----+--------------+-----+------+----+-----
+ 1 | 1 | [1, 10] | 1 | 1 | | 101
+ 1 | 1 | [1, 10] | 1 | null | | 101
+ 1 | 1 | [1, 10] | 1 | 2 | | 101
+ 1 | 1 | [1, 10] | 10 | 1 | | 110
+ 1 | 1 | [1, 10] | 10 | null | | 110
+ 1 | 1 | [1, 10] | 10 | 2 | | 110
+ 1 | 1 | [2] | 2 | 1 | | 102
+ 1 | 1 | [2] | 2 | null | | 102
+ 1 | 1 | [2] | 2 | 2 | | 102
+ 1 | 1 | [3, 30, 300] | 3 | 1 | | 103
+ 1 | 1 | [3, 30, 300] | 3 | null | | 103
+ 1 | 1 | [3, 30, 300] | 3 | 2 | | 103
+ 1 | 1 | [3, 30, 300] | 30 | 1 | | 130
+ 1 | 1 | [3, 30, 300] | 30 | null | | 130
+ 1 | 1 | [3, 30, 300] | 30 | 2 | | 130
+ 1 | 1 | [3, 30, 300] | 300 | 1 | | 400
+ 1 | 1 | [3, 30, 300] | 300 | null | | 400
+ 1 | 1 | [3, 30, 300] | 300 | 2 | | 400
+ 2 | 2 | | | | |
+ 3 | -1 | | | | |
+(20 rows)
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+ x | y | y | z
+---+---+--------------+---
+ 2 | 1 | [1, 2, 3] | 1
+ 2 | 1 | [1, 2, 3] | 2
+ 2 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [1, 2, 3] | 1
+ 3 | 1 | [1, 2, 3] | 2
+ 3 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [2, 3, 4, 5] | 2
+ 3 | 1 | [2, 3, 4, 5] | 3
+ 3 | 1 | [2, 3, 4, 5] | 4
+ 3 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [1, 2, 3] | 1
+ 4 | 1 | [1, 2, 3] | 2
+ 4 | 1 | [1, 2, 3] | 3
+ 4 | 1 | [2, 3, 4, 5] | 2
+ 4 | 1 | [2, 3, 4, 5] | 3
+ 4 | 1 | [2, 3, 4, 5] | 4
+ 4 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [3, 4, 5, 6] | 3
+ 4 | 1 | [3, 4, 5, 6] | 4
+ 4 | 1 | [3, 4, 5, 6] | 5
+ 4 | 1 | [3, 4, 5, 6] | 6
+ 2 | 2 | [1, 2, 3] | 2
+ 2 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [1, 2, 3] | 2
+ 3 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [2, 3, 4, 5] | 2
+ 3 | 2 | [2, 3, 4, 5] | 3
+ 3 | 2 | [2, 3, 4, 5] | 4
+ 3 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [1, 2, 3] | 2
+ 4 | 2 | [1, 2, 3] | 3
+ 4 | 2 | [2, 3, 4, 5] | 2
+ 4 | 2 | [2, 3, 4, 5] | 3
+ 4 | 2 | [2, 3, 4, 5] | 4
+ 4 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [3, 4, 5, 6] | 3
+ 4 | 2 | [3, 4, 5, 6] | 4
+ 4 | 2 | [3, 4, 5, 6] | 5
+ 4 | 2 | [3, 4, 5, 6] | 6
+ 2 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [2, 3, 4, 5] | 3
+ 3 | 3 | [2, 3, 4, 5] | 4
+ 3 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [1, 2, 3] | 3
+ 4 | 3 | [2, 3, 4, 5] | 3
+ 4 | 3 | [2, 3, 4, 5] | 4
+ 4 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [3, 4, 5, 6] | 3
+ 4 | 3 | [3, 4, 5, 6] | 4
+ 4 | 3 | [3, 4, 5, 6] | 5
+ 4 | 3 | [3, 4, 5, 6] | 6
+(52 rows)
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+ERROR: could not find jsonpath variable "x"
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+ json_value
+------------
+ foo
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_query
+------------
+ 123
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+ json_query
+------------
+ [123]
+(1 row)
+
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+ERROR: syntax error at or near " " of jsonpath input
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
+ERROR: only string constants supported in JSON_TABLE path specification
+LINE 1: SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || '...
+ ^
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
index 4f30fa46b9..df4a430d88 100644
--- a/src/test/regress/sql/json_sqljson.sql
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -9,3 +9,7 @@ SELECT JSON_VALUE(NULL FORMAT JSON, '$');
-- JSON_QUERY
SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+
+-- JSON_TABLE
+
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
index 77bfb6d61b..ea5db88b40 100644
--- a/src/test/regress/sql/jsonb_sqljson.sql
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -335,3 +335,677 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime()
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+
+-- JSON_TABLE
+
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- JSON_TABLE: Test backward parsing
+
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+
+\sv jsonb_table_view
+
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+
+-- JSON_TABLE: nested paths and plans
+
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+
+-- JSON_TABLE: plan validation
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+
+-- JSON_TABLE: plan execution
+
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9ee58ec2b0..a09df0e641 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1315,6 +1315,7 @@ JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
+JsonPathSpec
JsonPathString
JsonPathVarCallback
JsonPathVariable
@@ -1324,6 +1325,17 @@ JsonReturning
JsonScalarExpr
JsonSemAction
JsonSerializeExpr
+JsonTable
+JsonTableColumn
+JsonTableColumnType
+JsonTableContext
+JsonTableJoinState
+JsonTableParent
+JsonTablePlan
+JsonTablePlanJoinType
+JsonTablePlanType
+JsonTableScanState
+JsonTableSibling
JsonTokenType
JsonTransformStringValuesAction
JsonTypeCategory
@@ -2787,6 +2799,7 @@ TableFunc
TableFuncRoutine
TableFuncScan
TableFuncScanState
+TableFuncType
TableInfo
TableLikeClause
TableSampleClause
--
2.35.3
[application/octet-stream] v27-0003-SQL-JSON-query-functions.patch (196.5K, ../../CA+HiwqHgvd6akb4FxC7=GpmzXDAkGODXOBM+Q2g-UQ6+NK9sxw@mail.gmail.com/5-v27-0003-SQL-JSON-query-functions.patch)
download | inline diff:
From 2ffbec5408a8f3ccafc140dfc2db9767a61280ec Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Tue, 5 Dec 2023 14:33:25 +0900
Subject: [PATCH v27 3/4] SQL/JSON query functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This introduces the SQL/JSON functions for querying JSON data using
jsonpath expressions. The functions are:
JSON_EXISTS()
JSON_QUERY()
JSON_VALUE()
JSON_EXISTS() tests if the jsonpath expression applied to the jsonb
value yields any values.
JSON_VALUE() must return a single value, and an error occurs if it
tries to return multiple values.
JSON_QUERY() must return a json object or array, and there are
various WRAPPER options for handling scalar or multi-value results.
Both these functions have options for handling EMPTY and ERROR
conditions.
All of these functions only operate on jsonb. The workaround for now
is to cast the argument to jsonb.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Author: Peter Eisentraut <[email protected]>
Author: jian he <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he, Anton A. Melnikov, Nikita Malakhov, Peter Eisentraut
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 151 +++
src/backend/catalog/sql_features.txt | 12 +-
src/backend/executor/execExpr.c | 363 +++++++
src/backend/executor/execExprInterp.c | 367 ++++++-
src/backend/jit/llvm/llvmjit.c | 2 +
src/backend/jit/llvm/llvmjit_expr.c | 140 +++
src/backend/jit/llvm/llvmjit_types.c | 4 +
src/backend/nodes/makefuncs.c | 16 +
src/backend/nodes/nodeFuncs.c | 238 ++++-
src/backend/optimizer/path/costsize.c | 3 +-
src/backend/optimizer/util/clauses.c | 19 +
src/backend/parser/gram.y | 169 ++-
src/backend/parser/parse_expr.c | 644 +++++++++++-
src/backend/parser/parse_target.c | 15 +
src/backend/utils/adt/formatting.c | 44 +
src/backend/utils/adt/jsonb.c | 31 +
src/backend/utils/adt/jsonfuncs.c | 52 +-
src/backend/utils/adt/jsonpath.c | 255 +++++
src/backend/utils/adt/jsonpath_exec.c | 391 ++++++-
src/backend/utils/adt/ruleutils.c | 136 +++
src/include/executor/execExpr.h | 133 +++
src/include/fmgr.h | 1 +
src/include/jit/llvmjit.h | 1 +
src/include/nodes/makefuncs.h | 1 +
src/include/nodes/parsenodes.h | 47 +
src/include/nodes/primnodes.h | 130 +++
src/include/parser/kwlist.h | 11 +
src/include/utils/formatting.h | 1 +
src/include/utils/jsonb.h | 1 +
src/include/utils/jsonfuncs.h | 5 +
src/include/utils/jsonpath.h | 27 +
src/interfaces/ecpg/preproc/ecpg.trailer | 28 +
src/test/regress/expected/json_sqljson.out | 18 +
src/test/regress/expected/jsonb_sqljson.out | 1032 +++++++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/json_sqljson.sql | 11 +
src/test/regress/sql/jsonb_sqljson.sql | 337 ++++++
src/tools/pgindent/typedefs.list | 19 +
38 files changed, 4781 insertions(+), 76 deletions(-)
create mode 100644 src/test/regress/expected/json_sqljson.out
create mode 100644 src/test/regress/expected/jsonb_sqljson.out
create mode 100644 src/test/regress/sql/json_sqljson.sql
create mode 100644 src/test/regress/sql/jsonb_sqljson.sql
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 20da3ed033..7962a8a1a4 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -18156,6 +18156,157 @@ $.* ? (@ like_regex "^\\d+$")
</programlisting>
</para>
</sect3>
+
+ <sect3 id="sqljson-query-functions">
+ <title>SQL/JSON Query Functions</title>
+ <para>
+ <xref linkend="functions-sqljson-querying"/> details the SQL/JSON
+ functions that can be used to query JSON data.
+ </para>
+
+ <note>
+ <para>
+ SQL/JSON path expression can currently only accept values of the
+ <type>jsonb</type> type, so it might be necessary to cast the
+ <replaceable>context_item</replaceable> argument of these functions to
+ <type>jsonb</type>.
+ </para>
+ </note>
+
+ <table id="functions-sqljson-querying">
+ <title>SQL/JSON Query Functions</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ Function signature
+ </para>
+ <para>
+ Description
+ </para>
+ <para>
+ Example(s)
+ </para></entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_exists</primary></indexterm>
+ <function>json_exists</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> { <literal>TRUE</literal> | <literal>FALSE</literal> |<literal> UNKNOWN</literal> | <literal>ERROR</literal> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns true if the SQL/JSON <replaceable>path_expression</replaceable>
+ applied to the <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s yields any items.
+ The <literal>ON ERROR</literal> clause specifies what is returned if
+ an error occurs; the default is to return <literal>FALSE</literal>.
+ Note that if the <replaceable>path_expression</replaceable>
+ is <literal>strict</literal>, an error is generated if it yields no items.
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"key1": [1,2,3]}', 'strict $.key1[*] ? (@ > 2)')</literal>
+ <returnvalue>t</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'lax $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>f</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'strict $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>ERROR: jsonpath array subscript is out of bounds</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_query</primary></indexterm>
+ <function>json_query</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> <optional> <literal>FORMAT JSON</literal> <optional> <literal>ENCODING UTF8</literal> </optional> </optional> </optional>
+ <optional> { <literal>WITHOUT</literal> | <literal>WITH</literal> { <literal>CONDITIONAL</literal> | <optional><literal>UNCONDITIONAL</literal></optional> } } <optional> <literal>ARRAY</literal> </optional> <literal>WRAPPER</literal> </optional>
+ <optional> { <literal>KEEP</literal> | <literal>OMIT</literal> } <literal>QUOTES</literal> <optional> <literal>ON SCALAR STRING</literal> </optional> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s.
+ This function must return a JSON string, so if the path expression
+ returns multiple SQL/JSON items, you must wrap the result using the
+ <literal>WITH WRAPPER</literal> clause. If the wrapper is
+ <literal>UNCONDITIONAL</literal>, an array wrapper will always
+ be applied, even if the returned value is already a single JSON object
+ or an array, but if it is <literal>CONDITIONAL</literal>, it will not be
+ applied to a single array or object. <literal>UNCONDITIONAL</literal>
+ is the default. If the result is a scalar string, by default the value
+ returned will have surrounding quotes making it a valid JSON value,
+ which can be made explicit by specifying <literal>KEEP QUOTES</literal>.
+ Conversely, quotes can be omitted by specifying <literal>OMIT QUOTES</literal>.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>;
+ the default returned type is <type>jsonb</type>.
+ The <literal>ON EMPTY</literal> clause specifies the behavior if the
+ <replaceable>path_expression</replaceable> yields no value at all; the
+ default when <literal>ON EMPTY</literal> is not specified is to return a
+ null value. The <literal>ON ERROR</literal> clause specifies the behavior
+ if an error occurs as a result of <type>jsonpath</type> evaluation
+ (including cast to the output type) or during the execution of
+ <literal>ON EMPTY</literal> behavior (that was caused by empty result of
+ <type>jsonpath</type> evaluation); the default when
+ <literal>ON ERROR</literal> is not specified is to return a null value.
+ </para>
+ <para>
+ <literal>json_query(jsonb '[1,[2,3],null]', 'lax $[*][1]' WITH CONDITIONAL WRAPPER)</literal>
+ <returnvalue>[3]</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_value</primary></indexterm>
+ <function>json_value</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable>
+ <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <literal>PASSING</literal> <replaceable>value</replaceable>s. The
+ extracted value must be a single <acronym>SQL/JSON</acronym> scalar
+ item. For results that are objects or arrays, use the
+ <function>json_query</function> function instead.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>.
+ The default returned type is <type>text</type>.
+ The <literal>ON ERROR</literal> and <literal>ON EMPTY</literal>
+ clauses have similar semantics as mentioned in the description of
+ <function>json_query</function>.
+ </para>
+ <para>
+ <literal>json_value(jsonb '"123.45"', '$' RETURNING float)</literal>
+ <returnvalue>123.45</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '"03:04 2015-02-01"', '$.datetime("HH24:MI YYYY-MM-DD")' RETURNING date)</literal>
+ <returnvalue>2015-02-01</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '[1,2]', 'strict $[*]' DEFAULT 9 ON ERROR)</literal>
+ <returnvalue>9</returnvalue>
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect3>
</sect2>
</sect1>
diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt
index 80c40eaf57..7598bd8f22 100644
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -548,15 +548,15 @@ T811 Basic SQL/JSON constructor functions YES
T812 SQL/JSON: JSON_OBJECTAGG YES
T813 SQL/JSON: JSON_ARRAYAGG with ORDER BY YES
T814 Colon in JSON_OBJECT or JSON_OBJECTAGG YES
-T821 Basic SQL/JSON query operators NO
+T821 Basic SQL/JSON query operators YES
T822 SQL/JSON: IS JSON WITH UNIQUE KEYS predicate YES
-T823 SQL/JSON: PASSING clause NO
+T823 SQL/JSON: PASSING clause YES
T824 JSON_TABLE: specific PLAN clause NO
-T825 SQL/JSON: ON EMPTY and ON ERROR clauses NO
-T826 General value expression in ON ERROR or ON EMPTY clauses NO
+T825 SQL/JSON: ON EMPTY and ON ERROR clauses YES
+T826 General value expression in ON ERROR or ON EMPTY clauses YES
T827 JSON_TABLE: sibling NESTED COLUMNS clauses NO
-T828 JSON_QUERY NO
-T829 JSON_QUERY: array wrapper options NO
+T828 JSON_QUERY YES
+T829 JSON_QUERY: array wrapper options YES
T830 Enforcing unique keys in SQL/JSON constructor functions YES
T831 SQL/JSON path language: strict mode YES
T832 SQL/JSON path language: item method YES
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 34bd2102b5..b367c64a00 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -49,6 +49,7 @@
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/typcache.h"
@@ -88,6 +89,12 @@ static void ExecBuildAggTransCall(ExprState *state, AggState *aggstate,
FunctionCallInfo fcinfo, AggStatePerTrans pertrans,
int transno, int setno, int setoff, bool ishash,
bool nullcheck);
+static void ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch);
+static int ExecInitJsonExprCoercion(ExprState *state, Node *coercion,
+ ErrorSaveContext *escontext,
+ Datum *resv, bool *resnull);
/*
@@ -2416,6 +2423,36 @@ ExecInitExprRec(Expr *node, ExprState *state,
break;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+
+ ExecInitJsonExpr(jexpr, state, resv, resnull, &scratch);
+ break;
+ }
+
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = castNode(JsonCoercion, node);
+ JsonCoercionState *jcstate = palloc0(sizeof(JsonCoercionState));
+ Oid typinput;
+ FmgrInfo *finfo;
+
+ getTypeInputInfo(coercion->targettype, &typinput,
+ &jcstate->input.typioparam);
+ finfo = palloc0(sizeof(FmgrInfo));
+ fmgr_info(typinput, finfo);
+ jcstate->input.finfo = finfo;
+
+ jcstate->coercion = coercion;
+ jcstate->escontext = state->escontext;
+
+ scratch.opcode = EEOP_JSONEXPR_COERCION;
+ scratch.d.jsonexpr_coercion.jcstate = jcstate;
+ ExprEvalPushStep(state, &scratch);
+ break;
+ }
+
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -4184,3 +4221,329 @@ ExecBuildParamSetEqual(TupleDesc desc,
return state;
}
+
+/*
+ * Push steps to evaluate a JsonExpr and its various subsidiary expressions.
+ */
+static void
+ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch)
+{
+ JsonExprState *jsestate = palloc0(sizeof(JsonExprState));
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ ListCell *argexprlc;
+ ListCell *argnamelc;
+ List *jumps_if_skip = NIL;
+ List *jumps_to_coerce_finish = NIL;
+ List *jumps_to_end = NIL;
+ ListCell *lc;
+ ExprEvalStep *as;
+
+ jsestate->jsexpr = jexpr;
+
+ /*
+ * Evaluate formatted_expr storing the result into
+ * jsestate->formatted_expr.
+ */
+ ExecInitExprRec((Expr *) jexpr->formatted_expr, state,
+ &jsestate->formatted_expr.value,
+ &jsestate->formatted_expr.isnull);
+
+ /* Steps to jump to end if formatted_expr evaluates to NULL */
+ scratch->opcode = EEOP_JUMP_IF_NULL;
+ scratch->resnull = &jsestate->formatted_expr.isnull;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_if_skip = lappend_int(jumps_if_skip, state->steps_len);
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Evaluate pathspec expression storing the result into
+ * jsestate->pathspec.
+ */
+ ExecInitExprRec((Expr *) jexpr->path_spec, state,
+ &jsestate->pathspec.value,
+ &jsestate->pathspec.isnull);
+
+ /* Steps to JUMP to end if pathspec evaluates to NULL */
+ scratch->opcode = EEOP_JUMP_IF_NULL;
+ scratch->resnull = &jsestate->pathspec.isnull;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_if_skip = lappend_int(jumps_if_skip, state->steps_len);
+ ExprEvalPushStep(state, scratch);
+
+ /* Steps to compute PASSING args. */
+ jsestate->args = NIL;
+ forboth(argexprlc, jexpr->passing_values,
+ argnamelc, jexpr->passing_names)
+ {
+ Expr *argexpr = (Expr *) lfirst(argexprlc);
+ String *argname = lfirst_node(String, argnamelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = argname->sval;
+ var->typid = exprType((Node *) argexpr);
+ var->typmod = exprTypmod((Node *) argexpr);
+
+ ExecInitExprRec((Expr *) argexpr, state, &var->value, &var->isnull);
+
+ jsestate->args = lappend(jsestate->args, var);
+ }
+
+ /* Step for JsonPath* evaluation; see ExecEvalJsonExprPath(). */
+ scratch->opcode = EEOP_JSONEXPR_PATH;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Step to jump to end when there's neither an error when evaluating
+ * JsonPath* nor any need to coerce the result because it's already
+ * of the specified type.
+ */
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Steps to coerce the result value computed by EEOP_JSONEXPR_PATH.
+ * To handle coercion errors softly, use the following ErrorSaveContext
+ * when initializing the coercion expressions, including any JsonCoercion
+ * nodes.
+ */
+ jsestate->escontext.type = T_ErrorSaveContext;
+ if (jexpr->result_coercion || jexpr->omit_quotes)
+ {
+ jsestate->jump_eval_result_coercion =
+ ExecInitJsonExprCoercion(state, jexpr->result_coercion,
+ jexpr->on_error->btype != JSON_BEHAVIOR_ERROR ?
+ &jsestate->escontext : NULL,
+ resv, resnull);
+ }
+ else
+ jsestate->jump_eval_result_coercion = -1;
+
+ /* Steps for coercing JsonItemType values returned by JsonPathValue(). */
+ if (jexpr->item_coercions)
+ {
+ /*
+ * Jump to COERCION_FINISH to skip over the following steps if
+ * result_coercion is present.
+ */
+ if (jsestate->jump_eval_result_coercion >= 0)
+ {
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_to_coerce_finish = lappend_int(jumps_to_coerce_finish,
+ state->steps_len);
+ ExprEvalPushStep(state, scratch);
+ }
+
+ /*
+ * Here we create the steps for each JsonItemType type's coercion
+ * expression and also store a flag whether the expression is
+ * a JsonCoercion node. ExecPrepareJsonItemCoercion() called by
+ * ExecEvalJsonExprPath() will map a given JsonbValue returned by
+ * JsonPathValue() to its JsonItemType's expression's step address
+ * and the flag by indexing the following arrays with JsonItemType
+ * enum value.
+ */
+ jsestate->num_item_coercions = list_length(jexpr->item_coercions);
+ jsestate->eval_item_coercion_jumps = (int *)
+ palloc(jsestate->num_item_coercions * sizeof(int));
+ jsestate->item_coercion_via_expr = (bool *)
+ palloc0(jsestate->num_item_coercions * sizeof(bool));
+ foreach(lc, jexpr->item_coercions)
+ {
+ JsonItemCoercion *item_coercion = lfirst(lc);
+ Node *coercion = item_coercion->coercion;
+
+ jsestate->item_coercion_via_expr[item_coercion->item_type] =
+ (coercion != NULL && !IsA(coercion, JsonCoercion));
+ jsestate->eval_item_coercion_jumps[item_coercion->item_type] =
+ ExecInitJsonExprCoercion(state, coercion,
+ jexpr->on_error->btype != JSON_BEHAVIOR_ERROR ?
+ &jsestate->escontext : NULL,
+ resv, resnull);
+
+ /* Emit JUMP step to skip past other coercions' steps. */
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_to_coerce_finish = lappend_int(jumps_to_coerce_finish,
+ state->steps_len);
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /*
+ * Add step to reset the ErrorSaveContext and set error flag if the
+ * coercion steps encountered an error but was not thrown because of the
+ * ON ERROR behavior.
+ */
+ if (jexpr->result_coercion || jexpr->item_coercions)
+ {
+ foreach(lc, jumps_to_coerce_finish)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+
+ scratch->opcode = EEOP_JSONEXPR_COERCION_FINISH;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+ }
+
+ /*
+ * Step to handle ON ERROR behaviors. This handles both the errors
+ * that occur during EEOP_JSONEXPR_PATH evaluation and subsequent coercion
+ * evaluation.
+ */
+ jsestate->jump_error = -1;
+ if (jexpr->on_error &&
+ jexpr->on_error->btype != JSON_BEHAVIOR_ERROR)
+ {
+ jsestate->jump_error = state->steps_len;
+ scratch->opcode = EEOP_JUMP_IF_NOT_TRUE;
+
+ /*
+ * post_eval.error is set by ExecEvalJsonExprPath() and
+ * ExecEvalJsonCoercionFinish().
+ */
+ scratch->resvalue = &post_eval->error.value;
+ scratch->resnull = &post_eval->error.isnull;
+
+ scratch->d.jump.jumpdone = -1; /* set below */
+ ExprEvalPushStep(state, scratch);
+
+ /* Steps to evaluate the ON ERROR expression */
+ ExecInitExprRec((Expr *) jexpr->on_error->expr,
+ state, resv, resnull);
+
+ /* Steps to coerce the ON ERROR expression if needed */
+ if (jexpr->on_error->coercion)
+ ExecInitExprRec((Expr *) jexpr->on_error->coercion, state,
+ resv, resnull);
+
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+ }
+
+ /* Step to handle ON EMPTY behaviors. */
+ if (jexpr->on_empty != NULL &&
+ jexpr->on_empty->btype != JSON_BEHAVIOR_ERROR)
+ {
+ /*
+ * Make the ON ERROR behavior JUMP to here after checking the error
+ * and if it's not present then make EEOP_JSONEXPR_PATH directly
+ * jump here.
+ */
+ if (jsestate->jump_error >= 0)
+ {
+ as = &state->steps[jsestate->jump_error];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+ else
+ jsestate->jump_error = state->steps_len;
+
+ scratch->opcode = EEOP_JUMP_IF_NOT_TRUE;
+ scratch->resvalue = &post_eval->empty.value;
+ scratch->resnull = &post_eval->empty.isnull;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ ExprEvalPushStep(state, scratch);
+
+ /* Steps to evaluate the ON EMPTY expression */
+ ExecInitExprRec((Expr *) jexpr->on_empty->expr,
+ state, resv, resnull);
+
+ /* Steps to coerce the ON EMPTY expression if needed */
+ if (jexpr->on_empty->coercion)
+ ExecInitExprRec((Expr *) jexpr->on_empty->coercion, state,
+ resv, resnull);
+
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ ExprEvalPushStep(state, scratch);
+ }
+ /* Make EEOP_JSONEXPR_PATH jump to end if no ON EMPTY clause present. */
+ else if (jsestate->jump_error >= 0)
+ jumps_to_end = lappend_int(jumps_to_end, jsestate->jump_error);
+
+ /*
+ * If neither ON ERROR nor ON EMPTY jumps present, then add one to go
+ * to end.
+ */
+ if (jsestate->jump_error < 0)
+ {
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ ExprEvalPushStep(state, scratch);
+ }
+
+ /* Return NULL when either formatted_expr or pathspec is NULL. */
+ foreach(lc, jumps_if_skip)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+ scratch->opcode = EEOP_CONST;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.constval.value = (Datum) 0;
+ scratch->d.constval.isnull = true;
+ ExprEvalPushStep(state, scratch);
+
+ /* Jump to coerce the NULL using result_coercion is present. */
+ if (jsestate->jump_eval_result_coercion >= 0)
+ {
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = jsestate->jump_eval_result_coercion;
+ ExprEvalPushStep(state, scratch);
+ }
+
+ foreach(lc, jumps_to_end)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+}
+
+/* Initialize one JsonCoercion for execution. */
+static int
+ExecInitJsonExprCoercion(ExprState *state, Node *coercion,
+ ErrorSaveContext *escontext,
+ Datum *resv, bool *resnull)
+{
+ int jump_eval_coercion;
+ Datum *save_innermost_caseval;
+ bool *save_innermost_casenull;
+ ErrorSaveContext *save_escontext;
+
+ if (coercion == NULL)
+ return -1;
+
+ jump_eval_coercion = state->steps_len;
+
+ /* Push step(s) to compute cstate->coercion. */
+ save_innermost_caseval = state->innermost_caseval;
+ save_innermost_casenull = state->innermost_casenull;
+ save_escontext = state->escontext;
+
+ state->innermost_caseval = resv;
+ state->innermost_casenull = resnull;
+ state->escontext = escontext;
+
+ ExecInitExprRec((Expr *) coercion, state, resv, resnull);
+
+ state->innermost_caseval = save_innermost_caseval;
+ state->innermost_casenull = save_innermost_casenull;
+ state->escontext = save_escontext;
+
+ return jump_eval_coercion;
+}
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index d5db96444c..697b9a84a9 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -73,8 +73,8 @@
#include "utils/datum.h"
#include "utils/expandedrecord.h"
#include "utils/json.h"
-#include "utils/jsonb.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/timestamp.h"
@@ -181,6 +181,10 @@ static pg_attribute_always_inline void ExecAggPlainTransByRef(AggState *aggstate
AggStatePerGroup pergroup,
ExprContext *aggcontext,
int setno);
+static void ExecPrepareJsonItemCoercion(JsonbValue *item, JsonExprState *jsestate,
+ bool throw_error,
+ int *jump_eval_item_coercion,
+ Datum *resvalue, bool *resnull);
/*
* ScalarArrayOpExprHashEntry
@@ -482,6 +486,9 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_XMLEXPR,
&&CASE_EEOP_JSON_CONSTRUCTOR,
&&CASE_EEOP_IS_JSON,
+ &&CASE_EEOP_JSONEXPR_PATH,
+ &&CASE_EEOP_JSONEXPR_COERCION,
+ &&CASE_EEOP_JSONEXPR_COERCION_FINISH,
&&CASE_EEOP_AGGREF,
&&CASE_EEOP_GROUPING_FUNC,
&&CASE_EEOP_WINDOW_FUNC,
@@ -1551,6 +1558,35 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_JSONEXPR_PATH)
+ {
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+
+ /* too complex for an inline implementation */
+ if (!ExecEvalJsonExprPath(state, op, econtext))
+ EEO_JUMP(jsestate->jump_error);
+ else if (jsestate->post_eval.jump_eval_coercion >= 0)
+ EEO_JUMP(jsestate->post_eval.jump_eval_coercion);
+
+ EEO_NEXT();
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION)
+ {
+ /* too complex for an inline implementation */
+ ExecEvalJsonCoercion(state, op, econtext);
+
+ EEO_NEXT();
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION_FINISH)
+ {
+ /* too complex for an inline implementation */
+ ExecEvalJsonCoercionFinish(state, op);
+
+ EEO_NEXT();
+ }
+
EEO_CASE(EEOP_AGGREF)
{
/*
@@ -4208,6 +4244,335 @@ ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op)
*op->resvalue = BoolGetDatum(res);
}
+/*
+ * Performs JsonPath{Exists|Query|Value}() for given context item and JSON
+ * path.
+ *
+ * Result is set in *op->resvalue and *op->resnull.
+ *
+ * On return, JsonExprPostEvalState is populated with the following details:
+ * - jump_eval_coercion: step address of coercion to apply to the result
+ * - error.value: true if an error occurred during JsonPath evaluation
+ * - empty.value: true if JsonPath{Query|Value}() found no matching item
+ *
+ * No return if the ON ERROR/EMPTY behavior is ERROR.
+ */
+bool
+ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonExpr *jexpr = jsestate->jsexpr;
+ Datum item;
+ JsonPath *path;
+ bool throw_error = (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
+ bool error = false,
+ empty = false;
+
+ item = jsestate->formatted_expr.value;
+ path = DatumGetJsonPathP(jsestate->pathspec.value);
+
+ /* Reset JsonExprPostEvalState for this evaluation. */
+ memset(post_eval, 0, sizeof(*post_eval));
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ {
+ bool exists = JsonPathExists(item, path,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (!error)
+ {
+ *op->resvalue = BoolGetDatum(exists);
+ *op->resnull = false;
+ }
+ }
+ break;
+
+ case JSON_QUERY_OP:
+ *op->resvalue = JsonPathQuery(item, path, jexpr->wrapper, &empty,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (!error && !empty)
+ *op->resnull = (DatumGetPointer(*op->resvalue) == NULL);
+ break;
+
+ case JSON_VALUE_OP:
+ {
+ JsonbValue *jbv = JsonPathValue(item, path, &empty,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ /* Might get overridden by an item coercion below. */
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (jbv == NULL)
+ {
+ /* Will be coerced with result_coercion. */
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ }
+ else if (!error && !empty)
+ {
+ Assert(jbv != NULL);
+
+ /*
+ * If the requested output type is json(b), use
+ * result_coercion to do the coercion.
+ */
+ if (jexpr->returning->typid == JSONOID ||
+ jexpr->returning->typid == JSONBOID)
+ {
+ *op->resvalue = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ *op->resnull = false;
+ }
+ else
+ {
+ /*
+ * Else, use one of the item_coercions.
+ *
+ * Error out if no cast expression exists.
+ */
+ ExecPrepareJsonItemCoercion(jbv, jsestate, throw_error,
+ &post_eval->jump_eval_coercion,
+ op->resvalue, op->resnull);
+ }
+ }
+ break;
+ }
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
+ return false;
+ }
+
+ if (error)
+ {
+ Assert(!throw_error && jsestate->jump_error >= 0);
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ post_eval->error.value = BoolGetDatum(true);
+ return false;
+ }
+
+ if (empty)
+ {
+ if (jexpr->on_empty)
+ {
+ if (jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
+ ereport(ERROR,
+ (errcode(ERRCODE_NO_SQL_JSON_ITEM),
+ errmsg("no SQL/JSON item")));
+ else
+ post_eval->empty.value = BoolGetDatum(true);
+ }
+ else if (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR)
+ ereport(ERROR,
+ (errcode(ERRCODE_NO_SQL_JSON_ITEM),
+ errmsg("no SQL/JSON item")));
+ else
+ post_eval->error.value = BoolGetDatum(true);
+
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+
+ return false;
+ }
+
+ return true;
+}
+
+/*
+ * Selects a coercion for a given JsonbValue based on its type.
+ *
+ * On return, *resvalue and *resnull are set to the value extracted from the
+ * JsonbValue and *jump_eval_item_coercion is set to the step address of the
+ * coercion expression.
+ *
+ * If the found expression is a JsonCoercion node that means the parser
+ * didnt' find a cast to do the coercion, so throw an error if the
+ * ON ERROR behavior says to do so.
+ */
+static void
+ExecPrepareJsonItemCoercion(JsonbValue *item, JsonExprState *jsestate,
+ bool throw_error,
+ int *jump_eval_item_coercion,
+ Datum *resvalue, bool *resnull)
+{
+ int *eval_item_coercion_jumps = jsestate->eval_item_coercion_jumps;
+ bool *item_coercion_via_expr = jsestate->item_coercion_via_expr;
+ bool via_expr;
+ int jump_to;
+ JsonbValue buf;
+
+ if (item->type == jbvBinary && JsonContainerIsScalar(item->val.binary.data))
+ {
+ bool is_scalar PG_USED_FOR_ASSERTS_ONLY;
+
+ is_scalar = JsonbExtractScalar(item->val.binary.data, &buf);
+ item = &buf;
+ Assert(is_scalar);
+ }
+
+ *resnull = false;
+
+ /* get coercion state reference and datum of the corresponding SQL type */
+ switch (item->type)
+ {
+ case jbvNull:
+ via_expr = item_coercion_via_expr[JsonItemTypeNull];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeNull];
+ *resvalue = (Datum) 0;
+ *resnull = true;
+ break;
+
+ case jbvString:
+ via_expr = item_coercion_via_expr[JsonItemTypeString];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeString];
+ *resvalue =
+ PointerGetDatum(cstring_to_text_with_len(item->val.string.val,
+ item->val.string.len));
+ break;
+
+ case jbvNumeric:
+ via_expr = item_coercion_via_expr[JsonItemTypeNumeric];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeNumeric];
+ *resvalue = NumericGetDatum(item->val.numeric);
+ break;
+
+ case jbvBool:
+ via_expr = item_coercion_via_expr[JsonItemTypeBoolean];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeBoolean];
+ *resvalue = BoolGetDatum(item->val.boolean);
+ break;
+
+ case jbvDatetime:
+ *resvalue = item->val.datetime.value;
+ switch (item->val.datetime.typid)
+ {
+ case DATEOID:
+ via_expr = item_coercion_via_expr[JsonItemTypeDate];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeDate];
+ break;
+ case TIMEOID:
+ via_expr = item_coercion_via_expr[JsonItemTypeTime];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTime];
+ break;
+ case TIMETZOID:
+ via_expr = item_coercion_via_expr[JsonItemTypeTimetz];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimetz];
+ break;
+ case TIMESTAMPOID:
+ via_expr = item_coercion_via_expr[JsonItemTypeTimestamp];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimestamp];
+ break;
+ case TIMESTAMPTZOID:
+ via_expr = item_coercion_via_expr[JsonItemTypeTimestamptz];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimestamptz];
+ break;
+ default:
+ elog(ERROR, "unexpected jsonb datetime type oid %u",
+ item->val.datetime.typid);
+ }
+ break;
+
+ case jbvArray:
+ case jbvObject:
+ case jbvBinary:
+ via_expr = item_coercion_via_expr[JsonItemTypeComposite];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeComposite];
+ *resvalue = JsonbPGetDatum(JsonbValueToJsonb(item));
+ break;
+
+ default:
+ elog(ERROR, "unexpected jsonb value type %d", item->type);
+ }
+
+ /* If the expression is a JsonCoercion, throw an error. */
+ if (jump_to >= 0 && !via_expr)
+ {
+ if (throw_error)
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_ITEM_CANNOT_BE_CAST_TO_TARGET_TYPE),
+ errmsg("SQL/JSON item cannot be cast to target type")));
+
+ *resvalue = (Datum) 0;
+ *resnull = true;
+ }
+
+ *jump_eval_item_coercion = jump_to;
+}
+
+/*
+ * Coerce a jsonb value produced by ExecEvalJsonExprPath() or an ON ERROR /
+ * EMPTY behavior expression to the target type by either calling
+ * json_populate_type() or the type's input function.
+ *
+ * If a soft-error occurs, it will be checked by EEOP_JSONEXPR_COECION_FINISH
+ * that will run right after this.
+ */
+void
+ExecEvalJsonCoercion(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext)
+{
+ JsonCoercionState *jcstate = op->d.jsonexpr_coercion.jcstate;
+ JsonCoercion *coercion = jcstate->coercion;
+ Datum res = *op->resvalue;
+ bool resnull = *op->resnull;
+
+ if (coercion->via_populate)
+ {
+ *op->resvalue = json_populate_type(res, JSONBOID,
+ coercion->targettype,
+ coercion->targettypmod,
+ &jcstate->cache,
+ econtext->ecxt_per_query_memory,
+ op->resnull,
+ (Node *) jcstate->escontext);
+ }
+ else if (coercion->via_io)
+ {
+ char *val_string = resnull ? NULL :
+ JsonbUnquote(DatumGetJsonbP(res));
+
+ (void) InputFunctionCallSafe(jcstate->input.finfo, val_string,
+ jcstate->input.typioparam,
+ coercion->targettypmod,
+ (Node *) jcstate->escontext,
+ op->resvalue);
+ }
+}
+
+/*
+ * Checks if the coercion evaluation led to an error. If an error did occur,
+ * this sets post_eval->error to trigger the subsequent ON ERROR handling
+ * steps.
+ */
+void
+ExecEvalJsonCoercionFinish(ExprState *state, ExprEvalStep *op)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+
+ if (SOFT_ERROR_OCCURRED(&jsestate->escontext))
+ {
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ jsestate->post_eval.error.value = BoolGetDatum(true);
+
+ /*
+ * Also make ErrorSaveContext ready for the next row. Since we never
+ * set details_wanted, we don't need to also reset error_data, which
+ * would be NULL anyway.
+ */
+ Assert(!jsestate->escontext.details_wanted &&
+ jsestate->escontext.error_data == NULL);
+ jsestate->escontext.error_occurred = false;
+ }
+}
/*
* ExecEvalGroupingFunc
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 2c8ac02550..4e83e5ef7f 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -84,6 +84,7 @@ LLVMTypeRef StructAggState;
LLVMTypeRef StructAggStatePerGroupData;
LLVMTypeRef StructAggStatePerTransData;
LLVMTypeRef StructPlanState;
+LLVMTypeRef StructJsonExprPostEvalState;
LLVMValueRef AttributeTemplate;
LLVMValueRef ExecEvalSubroutineTemplate;
@@ -1186,6 +1187,7 @@ llvm_create_types(void)
StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
StructPlanState = llvm_pg_var_type("StructPlanState");
StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
+ StructJsonExprPostEvalState = llvm_pg_var_type("StructJsonExprPostEvalState");
AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index 81856a9dc7..5c67d7749b 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1930,6 +1930,146 @@ llvm_compile_expr(ExprState *state)
LLVMBuildBr(b, opblocks[opno + 1]);
break;
+ case EEOP_JSONEXPR_PATH:
+ {
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ LLVMValueRef v_ret;
+ LLVMBasicBlockRef b_coercion;
+
+ /*
+ * Call ExecEvalJsonExprPath(). It returns false when
+ * an error occurs or if the no match is found, which
+ * is handled by jumping to the block at
+ * jsestate->jump_error. If true is returned, jump
+ * to the block that coerces the result value.
+ */
+ v_ret = build_EvalXFunc(b, mod, "ExecEvalJsonExprPath",
+ v_state, op, v_econtext);
+ v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, "");
+
+ b_coercion =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_coercion", opno);
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_sbool_const(0),
+ ""),
+ jsestate->jump_error >= 0 ?
+ opblocks[jsestate->jump_error] :
+ b_coercion,
+ b_coercion);
+
+ /*
+ * Build a switch to map
+ * JsonExprPostEvalState.jump_eval_coercion, which is a
+ * runtime value of the step address of the coercion
+ * expression set by ExecEvalJsonExprPath() based on the
+ * result value it computes, to either
+ * jsestate->jump_eval_result_coercion or one of those in
+ * the jsestate->eval_item_coercion_jumps[] array.
+ */
+ LLVMPositionBuilderAtEnd(b, b_coercion);
+ if (jsestate->jump_eval_result_coercion >= 0 ||
+ jsestate->num_item_coercions > 0)
+ {
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ int i;
+ LLVMValueRef v_post_eval;
+ LLVMValueRef v_post_eval_jump_eval_coercionp;
+ LLVMValueRef v_post_eval_jump_eval_coercion;
+ LLVMValueRef v_jump_coercion;
+ LLVMValueRef v_switch;
+ LLVMBasicBlockRef b_done,
+ b_result_coercion_block,
+ *b_item_coercion_blocks = NULL;
+
+ v_post_eval = l_ptr_const(post_eval, l_ptr(StructJsonExprPostEvalState));
+ v_post_eval_jump_eval_coercionp =
+ l_struct_gep(b,
+ StructJsonExprPostEvalState,
+ v_post_eval,
+ FIELDNO_JSONEXPRPOSTEVALSTATE_JUMP_EVAL_COERCION,
+ "v_post_eval_jump_eval_coercion");
+ v_post_eval_jump_eval_coercion =
+ l_load(b, LLVMInt32TypeInContext(lc),
+ v_post_eval_jump_eval_coercionp, "");
+
+ b_result_coercion_block =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_result_coercion", opno);
+ if (jsestate->num_item_coercions > 0)
+ {
+ b_item_coercion_blocks = palloc(sizeof(LLVMBasicBlockRef) *
+ jsestate->num_item_coercions);
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ b_item_coercion_blocks[i] =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_item_coercion.%d",
+ opno, i);
+ }
+ }
+ b_done =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_done", opno);
+
+ v_switch = LLVMBuildSwitch(b,
+ v_post_eval_jump_eval_coercion,
+ b_done,
+ jsestate->num_item_coercions + 1);
+ if (jsestate->jump_eval_result_coercion >= 0)
+ {
+ v_jump_coercion = l_int32_const(lc, jsestate->jump_eval_result_coercion);
+ LLVMAddCase(v_switch, v_jump_coercion, b_result_coercion_block);
+ }
+
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ if (jsestate->eval_item_coercion_jumps[i] >= 0)
+ {
+ v_jump_coercion = l_int32_const(lc, jsestate->eval_item_coercion_jumps[i]);
+ LLVMAddCase(v_switch, v_jump_coercion, b_item_coercion_blocks[i]);
+ }
+ }
+
+ LLVMPositionBuilderAtEnd(b, b_result_coercion_block);
+ if (jsestate->jump_eval_result_coercion >= 0)
+ LLVMBuildBr(b, opblocks[jsestate->jump_eval_result_coercion]);
+ else
+ LLVMBuildUnreachable(b);
+
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ LLVMPositionBuilderAtEnd(b, b_item_coercion_blocks[i]);
+ if (jsestate->eval_item_coercion_jumps[i] >= 0)
+ LLVMBuildBr(b, opblocks[jsestate->eval_item_coercion_jumps[i]]);
+ else
+ LLVMBuildUnreachable(b);
+ }
+
+ LLVMPositionBuilderAtEnd(b, b_done);
+ }
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+ }
+
+ case EEOP_JSONEXPR_COERCION:
+ build_EvalXFunc(b, mod, "ExecEvalJsonCoercion",
+ v_state, op, v_econtext);
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
+ case EEOP_JSONEXPR_COERCION_FINISH:
+ build_EvalXFunc(b, mod, "ExecEvalJsonCoercionFinish",
+ v_state, op);
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
case EEOP_AGGREF:
{
LLVMValueRef v_aggno;
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 3a4be09e50..1546037b3a 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -69,6 +69,7 @@ MinimalTupleTableSlot StructMinimalTupleTableSlot;
TupleDescData StructTupleDescData;
PlanState StructPlanState;
MinimalTupleData StructMinimalTupleData;
+JsonExprPostEvalState StructJsonExprPostEvalState;
/*
@@ -172,6 +173,9 @@ void *referenced_functions[] =
ExecEvalXmlExpr,
ExecEvalJsonConstructor,
ExecEvalJsonIsPredicate,
+ ExecEvalJsonCoercion,
+ ExecEvalJsonCoercionFinish,
+ ExecEvalJsonExprPath,
MakeExpandedObjectReadOnlyInternal,
slot_getmissingattrs,
slot_getsomeattrs_int,
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index c6fb571982..d5bb5cded6 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -857,6 +857,22 @@ makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
return jve;
}
+/*
+ * makeJsonBehavior -
+ * creates a JsonBehavior node
+ */
+JsonBehavior *
+makeJsonBehavior(JsonBehaviorType type, Node *expr, int location)
+{
+ JsonBehavior *behavior = makeNode(JsonBehavior);
+
+ behavior->btype = type;
+ behavior->expr = expr;
+ behavior->location = location;
+
+ return behavior;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index c03f4f23e2..62bf08ad27 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -234,6 +234,33 @@ exprType(const Node *expr)
case T_JsonIsPredicate:
type = BOOLOID;
break;
+ case T_JsonExpr:
+ {
+ const JsonExpr *jexpr = (const JsonExpr *) expr;
+
+ type = jexpr->returning->typid;
+ break;
+ }
+ case T_JsonCoercion:
+ {
+ const JsonCoercion *coercion = (const JsonCoercion *) expr;
+
+ type = coercion->targettype;
+ break;
+ }
+ case T_JsonItemCoercion:
+ type = exprType(((JsonItemCoercion *) expr)->coercion);
+ break;
+ case T_JsonBehavior:
+ {
+ const JsonBehavior *behavior = (const JsonBehavior *) expr;
+
+ if (behavior->coercion)
+ type = exprType((Node *) behavior->coercion);
+ else
+ type = exprType(behavior->expr);
+ break;
+ }
case T_NullTest:
type = BOOLOID;
break;
@@ -491,8 +518,32 @@ exprTypmod(const Node *expr)
return ((const SQLValueFunction *) expr)->typmod;
case T_JsonValueExpr:
return exprTypmod((Node *) ((const JsonValueExpr *) expr)->formatted_expr);
- case T_JsonConstructorExpr:
- return ((const JsonConstructorExpr *) expr)->returning->typmod;
+ case T_JsonExpr:
+ {
+ const JsonExpr *jexpr = (const JsonExpr *) expr;
+
+ return jexpr->returning->typmod;
+ }
+ break;
+ case T_JsonCoercion:
+ {
+ const JsonCoercion *coercion = (const JsonCoercion *) expr;
+
+ return coercion->targettypmod;
+ }
+ break;
+ case T_JsonItemCoercion:
+ return exprTypmod(((JsonItemCoercion *) expr)->coercion);
+ case T_JsonBehavior:
+ {
+ const JsonBehavior *behavior = (const JsonBehavior *) expr;
+
+ if (behavior->coercion)
+ return exprTypmod((Node *) behavior->coercion);
+ else
+ return exprTypmod(behavior->expr);
+ }
+ break;
case T_CoerceToDomain:
return ((const CoerceToDomain *) expr)->resulttypmod;
case T_CoerceToDomainValue:
@@ -969,6 +1020,34 @@ exprCollation(const Node *expr)
/* IS JSON's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
break;
+ case T_JsonExpr:
+ coll = exprCollation(((JsonExpr *) expr)->result_coercion);
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) expr;
+
+ if (coercion->via_io || coercion->via_populate)
+ coll = coercion->collation;
+ else
+ coll = InvalidOid;
+ }
+ break;
+ case T_JsonItemCoercion:
+ coll = exprCollation(((JsonItemCoercion *) expr)->coercion);
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *behavior = (JsonBehavior *) expr;
+
+ if (behavior->coercion)
+ coll = exprCollation((Node *) behavior->coercion);
+ else if (behavior->expr)
+ coll = exprCollation(behavior->expr);
+ else
+ coll = InvalidOid;
+ }
+ break;
case T_NullTest:
/* NullTest's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
@@ -1205,6 +1284,42 @@ exprSetCollation(Node *expr, Oid collation)
case T_JsonIsPredicate:
Assert(!OidIsValid(collation)); /* result is always boolean */
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) expr;
+
+ if (jexpr->result_coercion)
+ exprSetCollation((Node *) jexpr->result_coercion, collation);
+ else
+ Assert(!OidIsValid(collation)); /* result is always a
+ * json[b] type */
+ }
+ break;
+ case T_JsonItemCoercion:
+ {
+ JsonItemCoercion *item_coercion = (JsonItemCoercion *) expr;
+
+ if (item_coercion->coercion)
+ exprSetCollation(item_coercion->coercion, collation);
+ }
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) expr;
+
+ coercion->collation = collation;
+ }
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *behavior = (JsonBehavior *) expr;
+
+ if (behavior->expr)
+ exprSetCollation(behavior->expr, collation);
+ if (behavior->coercion)
+ exprSetCollation((Node *) behavior->coercion, collation);
+ }
+ break;
case T_NullTest:
/* NullTest's result is boolean ... */
Assert(!OidIsValid(collation)); /* ... so never set a collation */
@@ -1508,6 +1623,18 @@ exprLocation(const Node *expr)
case T_JsonIsPredicate:
loc = ((const JsonIsPredicate *) expr)->location;
break;
+ case T_JsonExpr:
+ {
+ const JsonExpr *jsexpr = (const JsonExpr *) expr;
+
+ /* consider both function name and leftmost arg */
+ loc = leftmostLoc(jsexpr->location,
+ exprLocation(jsexpr->formatted_expr));
+ }
+ break;
+ case T_JsonBehavior:
+ loc = exprLocation(((JsonBehavior *) expr)->expr);
+ break;
case T_NullTest:
{
const NullTest *nexpr = (const NullTest *) expr;
@@ -2260,6 +2387,45 @@ expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ if (WALK(jexpr->formatted_expr))
+ return true;
+ if (WALK(jexpr->result_coercion))
+ return true;
+ if (WALK(jexpr->item_coercions))
+ return true;
+ if (WALK(jexpr->passing_values))
+ return true;
+ /* we assume walker doesn't care about passing_names */
+ if (WALK(jexpr->on_empty))
+ return true;
+ if (WALK(jexpr->on_error))
+ return true;
+ }
+ break;
+ case T_JsonCoercion:
+ break;
+ case T_JsonItemCoercion:
+ {
+ JsonItemCoercion *item_coercion = (JsonItemCoercion *) node;
+
+ if (WALK(item_coercion->coercion))
+ return true;
+ }
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *behavior = (JsonBehavior *) node;
+
+ if (WALK(behavior->expr))
+ return true;
+ if (WALK(behavior->coercion))
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
@@ -3259,6 +3425,46 @@ expression_tree_mutator_impl(Node *node,
return (Node *) newnode;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+ JsonExpr *newnode;
+
+ FLATCOPY(newnode, jexpr, JsonExpr);
+ MUTATE(newnode->path_spec, jexpr->path_spec, Node *);
+ MUTATE(newnode->formatted_expr, jexpr->formatted_expr, Node *);
+ MUTATE(newnode->result_coercion, jexpr->result_coercion, Node *);
+ MUTATE(newnode->item_coercions, jexpr->item_coercions, List *);
+ MUTATE(newnode->passing_values, jexpr->passing_values, List *);
+ /* assume mutator does not care about passing_names */
+ MUTATE(newnode->on_empty, jexpr->on_empty, JsonBehavior *);
+ MUTATE(newnode->on_error, jexpr->on_error, JsonBehavior *);
+ return (Node *) newnode;
+ }
+ break;
+ case T_JsonCoercion:
+ return (Node *) copyObject(node);
+ case T_JsonItemCoercion:
+ {
+ JsonItemCoercion *item_coercion = (JsonItemCoercion *) node;
+ JsonItemCoercion *newnode;
+
+ FLATCOPY(newnode, item_coercion, JsonItemCoercion);
+ MUTATE(newnode->coercion, item_coercion->coercion, Node *);
+ return (Node *) newnode;
+ }
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *behavior = (JsonBehavior *) node;
+ JsonBehavior *newnode;
+
+ FLATCOPY(newnode, behavior, JsonBehavior);
+ MUTATE(newnode->expr, behavior->expr, Node *);
+ MUTATE(newnode->coercion, behavior->coercion, JsonCoercion *);
+ return (Node *) newnode;
+ }
+ break;
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -3945,6 +4151,34 @@ raw_expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonArgument:
+ return WALK(((JsonArgument *) node)->val);
+ case T_JsonFuncExpr:
+ {
+ JsonFuncExpr *jfe = (JsonFuncExpr *) node;
+
+ if (WALK(jfe->context_item))
+ return true;
+ if (WALK(jfe->pathspec))
+ return true;
+ if (WALK(jfe->passing))
+ return true;
+ if (jfe->output && WALK(jfe->output))
+ return true;
+ if (jfe->behavior)
+ return true;
+ }
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *jb = (JsonBehavior *) node;
+
+ if (WALK(jb->expr))
+ return true;
+ if (WALK(jb->coercion))
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index d6ceafd51c..3bb48b64d5 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -4850,7 +4850,8 @@ cost_qual_eval_walker(Node *node, cost_qual_eval_context *context)
IsA(node, SQLValueFunction) ||
IsA(node, XmlExpr) ||
IsA(node, CoerceToDomain) ||
- IsA(node, NextValueExpr))
+ IsA(node, NextValueExpr) ||
+ IsA(node, JsonExpr))
{
/* Treat all these as having cost 1 */
context->total.per_tuple += cpu_operator_cost;
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index 507c101661..06297b0391 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -53,6 +53,7 @@
#include "utils/fmgroids.h"
#include "utils/json.h"
#include "utils/jsonb.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/syscache.h"
@@ -417,6 +418,24 @@ contain_mutable_functions_walker(Node *node, void *context)
/* Check all subnodes */
}
+ if (IsA(node, JsonExpr))
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+ Const *cnst;
+
+ if (!IsA(jexpr->path_spec, Const))
+ return true;
+
+ cnst = castNode(Const, jexpr->path_spec);
+
+ Assert(cnst->consttype == JSONPATHOID);
+ if (cnst->constisnull)
+ return false;
+
+ return jspIsMutable(DatumGetJsonPathP(cnst->constvalue),
+ jexpr->passing_names, jexpr->passing_values);
+ }
+
if (IsA(node, SQLValueFunction))
{
/* all variants of SQLValueFunction are stable */
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index d631ac89a9..e625b20b13 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -650,11 +650,18 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_returning_clause_opt
json_name_and_value
json_aggregate_func
+ json_argument
+ json_behavior
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
+ json_arguments
+ json_behavior_clause_opt
+ json_passing_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
+ json_quotes_clause_opt
+ json_wrapper_behavior
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
@@ -695,7 +702,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ COMMITTED COMPRESSION CONCURRENTLY CONDITIONAL CONFIGURATION CONFLICT
CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
@@ -706,8 +713,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
- EACH ELSE ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ESCAPE EVENT EXCEPT
- EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
+ EACH ELSE EMPTY_P ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ERROR_P ESCAPE
+ EVENT EXCEPT EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
EXTENSION EXTERNAL EXTRACT
FALSE_P FAMILY FETCH FILTER FINALIZE FIRST_P FLOAT_P FOLLOWING FOR
@@ -722,10 +729,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
- JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_OBJECT JSON_OBJECTAGG
- JSON_SCALAR JSON_SERIALIZE
+ JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
- KEY KEYS
+ KEEP KEY KEYS
LABEL LANGUAGE LARGE_P LAST_P LATERAL_P
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
@@ -739,7 +746,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
- OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR
+ OBJECT_P OF OFF OFFSET OIDS OLD OMIT ON ONLY OPERATOR OPTION OPTIONS OR
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
@@ -748,7 +755,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
- QUOTE
+ QUOTE QUOTES
RANGE READ REAL REASSIGN RECHECK RECURSIVE REF_P REFERENCES REFERENCING
REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA
@@ -759,7 +766,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
SEQUENCE SEQUENCES
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
- START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
+ START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
@@ -767,7 +774,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TREAT TRIGGER TRIM TRUE_P
TRUNCATE TRUSTED TYPE_P TYPES_P
- UESCAPE UNBOUNDED UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
+ UESCAPE UNBOUNDED UNCONDITIONAL UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
UNLISTEN UNLOGGED UNTIL UPDATE USER USING
VACUUM VALID VALIDATE VALIDATOR VALUE_P VALUES VARCHAR VARIADIC VARYING
@@ -15768,6 +15775,60 @@ func_expr_common_subexpr:
n->location = @1;
$$ = (Node *) n;
}
+ | JSON_QUERY '('
+ json_value_expr ',' a_expr json_passing_clause_opt
+ json_returning_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_QUERY_OP;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->wrapper = $8;
+ n->quotes = $9;
+ n->behavior = $10;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_EXISTS '('
+ json_value_expr ',' a_expr json_passing_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_EXISTS_OP;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = NULL;
+ n->behavior = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_VALUE '('
+ json_value_expr ',' a_expr json_passing_clause_opt
+ json_returning_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_VALUE_OP;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->behavior = $8;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
;
@@ -16494,6 +16555,27 @@ opt_asymmetric: ASYMMETRIC
;
/* SQL/JSON support */
+json_passing_clause_opt:
+ PASSING json_arguments { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
+ ;
+
+json_arguments:
+ json_argument { $$ = list_make1($1); }
+ | json_arguments ',' json_argument { $$ = lappend($1, $3); }
+ ;
+
+json_argument:
+ json_value_expr AS ColLabel
+ {
+ JsonArgument *n = makeNode(JsonArgument);
+
+ n->val = (JsonValueExpr *) $1;
+ n->name = $3;
+ $$ = (Node *) n;
+ }
+ ;
+
json_value_expr:
a_expr json_format_clause_opt
{
@@ -16519,6 +16601,27 @@ json_encoding_clause_opt:
| /* EMPTY */ { $$ = JS_ENC_DEFAULT; }
;
+/* ARRAY is a noise word */
+json_wrapper_behavior:
+ WITHOUT WRAPPER { $$ = JSW_NONE; }
+ | WITHOUT ARRAY WRAPPER { $$ = JSW_NONE; }
+ | WITH WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL ARRAY WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | /* empty */ { $$ = JSW_NONE; }
+ ;
+
+json_quotes_clause_opt:
+ KEEP QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_KEEP; }
+ | KEEP QUOTES { $$ = JS_QUOTES_KEEP; }
+ | OMIT QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_OMIT; }
+ | OMIT QUOTES { $$ = JS_QUOTES_OMIT; }
+ | /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -16532,6 +16635,30 @@ json_returning_clause_opt:
| /* EMPTY */ { $$ = NULL; }
;
+json_behavior:
+ DEFAULT a_expr { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
+ | ERROR_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | NULL_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
+ | TRUE_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
+ | FALSE_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
+ | UNKNOWN { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
+ | EMPTY_P ARRAY { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P OBJECT_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
+ /* non-standard, for Oracle compatibility only */
+ | EMPTY_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ ;
+
+json_behavior_clause_opt:
+ json_behavior ON EMPTY_P
+ { $$ = list_make2($1, NULL); }
+ | json_behavior ON ERROR_P
+ { $$ = list_make2(NULL, $1); }
+ | json_behavior ON EMPTY_P json_behavior ON ERROR_P
+ { $$ = list_make2($1, $4); }
+ | /* EMPTY */
+ { $$ = list_make2(NULL, NULL); }
+ ;
+
/*
* We must assign the only-JSON production a precedence less than IDENT in
* order to favor shifting over reduction when JSON is followed by VALUE_P,
@@ -17135,6 +17262,7 @@ unreserved_keyword:
| COMMIT
| COMMITTED
| COMPRESSION
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17171,10 +17299,12 @@ unreserved_keyword:
| DOUBLE_P
| DROP
| EACH
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17224,6 +17354,7 @@ unreserved_keyword:
| INSTEAD
| INVOKER
| ISOLATION
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17270,6 +17401,7 @@ unreserved_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| OPERATOR
| OPTION
| OPTIONS
@@ -17300,6 +17432,7 @@ unreserved_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REASSIGN
@@ -17359,6 +17492,7 @@ unreserved_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUPPORT
@@ -17381,6 +17515,7 @@ unreserved_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNKNOWN
| UNLISTEN
@@ -17441,10 +17576,13 @@ col_name_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
| LEAST
| NATIONAL
| NCHAR
@@ -17677,6 +17815,7 @@ bare_label_keyword:
| COMMITTED
| COMPRESSION
| CONCURRENTLY
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17729,11 +17868,13 @@ bare_label_keyword:
| DROP
| EACH
| ELSE
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| END_P
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17803,10 +17944,14 @@ bare_label_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17867,6 +18012,7 @@ bare_label_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| ONLY
| OPERATOR
| OPTION
@@ -17904,6 +18050,7 @@ bare_label_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REAL
@@ -17972,6 +18119,7 @@ bare_label_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUBSTRING
@@ -18006,6 +18154,7 @@ bare_label_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNIQUE
| UNKNOWN
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64c582c344..4379008b4c 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -37,6 +37,7 @@
#include "utils/builtins.h"
#include "utils/date.h"
#include "utils/fmgroids.h"
+#include "utils/jsonb.h"
#include "utils/lsyscache.h"
#include "utils/timestamp.h"
#include "utils/xml.h"
@@ -90,6 +91,23 @@ static Node *transformJsonParseExpr(ParseState *pstate, JsonParseExpr *expr);
static Node *transformJsonScalarExpr(ParseState *pstate, JsonScalarExpr *expr);
static Node *transformJsonSerializeExpr(ParseState *pstate,
JsonSerializeExpr *expr);
+static Node *transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *p);
+static JsonExpr *transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName);
+static void transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names);
+static Node *coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr);
+static Oid JsonFuncExprDefaultReturnType(JsonExpr *jsexpr);
+static Node *coerceJsonExpr(ParseState *pstate, Node *expr,
+ const JsonReturning *returning);
+static List *InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId);
+static JsonBehavior *transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning);
+static JsonBehavior *coerceJsonBehaviorExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning);
static Node *make_row_comparison_op(ParseState *pstate, List *opname,
List *largs, List *rargs, int location);
static Node *make_row_distinct_op(ParseState *pstate, List *opname,
@@ -353,6 +371,10 @@ transformExprRecurse(ParseState *pstate, Node *expr)
result = transformJsonSerializeExpr(pstate, (JsonSerializeExpr *) expr);
break;
+ case T_JsonFuncExpr:
+ result = transformJsonFuncExpr(pstate, (JsonFuncExpr *) expr);
+ break;
+
default:
/* should not reach here */
elog(ERROR, "unrecognized node type: %d", (int) nodeTag(expr));
@@ -3229,7 +3251,7 @@ makeJsonByteaToTextConversion(Node *expr, JsonFormat *format, int location)
static Node *
transformJsonValueExpr(ParseState *pstate, const char *constructName,
JsonValueExpr *ve, JsonFormatType default_format,
- Oid targettype)
+ Oid targettype, bool isarg)
{
Node *expr = transformExprRecurse(pstate, (Node *) ve->raw_expr);
Node *rawexpr;
@@ -3261,6 +3283,35 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
else
format = ve->format->format_type;
}
+ else if (isarg)
+ {
+ /* Pass SQL/JSON item types directly without conversion to json[b]. */
+ switch (exprtype)
+ {
+ case TEXTOID:
+ case NUMERICOID:
+ case BOOLOID:
+ case INT2OID:
+ case INT4OID:
+ case INT8OID:
+ case FLOAT4OID:
+ case FLOAT8OID:
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ return expr;
+
+ default:
+ if (typcategory == TYPCATEGORY_STRING)
+ return expr;
+ /* else convert argument to json[b] type */
+ break;
+ }
+
+ format = default_format;
+ }
else if (exprtype == JSONOID || exprtype == JSONBOID)
format = JS_FORMAT_DEFAULT; /* do not format json[b] types */
else
@@ -3272,7 +3323,8 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
Node *coerced;
bool only_allow_cast = OidIsValid(targettype);
- if (!only_allow_cast &&
+ if (!isarg &&
+ !only_allow_cast &&
exprtype != BYTEAOID && typcategory != TYPCATEGORY_STRING)
ereport(ERROR,
errcode(ERRCODE_DATATYPE_MISMATCH),
@@ -3425,6 +3477,11 @@ transformJsonOutput(ParseState *pstate, const JsonOutput *output,
errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("returning SETOF types is not supported in SQL/JSON functions"));
+ if (get_typtype(ret->typid) == TYPTYPE_PSEUDO)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("returning pseudo-types is not supported in SQL/JSON functions"));
+
if (ret->format->format_type == JS_FORMAT_DEFAULT)
/* assign JSONB format when returning jsonb, or JSON format otherwise */
ret->format->format_type =
@@ -3621,7 +3678,7 @@ transformJsonObjectConstructor(ParseState *pstate, JsonObjectConstructor *ctor)
Node *val = transformJsonValueExpr(pstate, "JSON_OBJECT()",
kv->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = lappend(args, key);
args = lappend(args, val);
@@ -3808,7 +3865,7 @@ transformJsonObjectAgg(ParseState *pstate, JsonObjectAgg *agg)
val = transformJsonValueExpr(pstate, "JSON_OBJECTAGG()",
agg->arg->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = list_make2(key, val);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
@@ -3864,9 +3921,8 @@ transformJsonArrayAgg(ParseState *pstate, JsonArrayAgg *agg)
Oid aggfnoid;
Oid aggtype;
- arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()",
- agg->arg,
- JS_FORMAT_DEFAULT, InvalidOid);
+ arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()", agg->arg,
+ JS_FORMAT_DEFAULT, InvalidOid, false);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
list_make1(arg));
@@ -3913,9 +3969,8 @@ transformJsonArrayConstructor(ParseState *pstate, JsonArrayConstructor *ctor)
{
JsonValueExpr *jsval = castNode(JsonValueExpr, lfirst(lc));
Node *val = transformJsonValueExpr(pstate, "JSON_ARRAY()",
- jsval,
- JS_FORMAT_DEFAULT,
- InvalidOid);
+ jsval, JS_FORMAT_DEFAULT,
+ InvalidOid, false);
args = lappend(args, val);
}
@@ -4074,7 +4129,7 @@ transformJsonParseExpr(ParseState *pstate, JsonParseExpr *jsexpr)
* function-like CASTs.
*/
arg = transformJsonValueExpr(pstate, "JSON()", jsexpr->expr,
- JS_FORMAT_JSON, returning->typid);
+ JS_FORMAT_JSON, returning->typid, false);
}
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_PARSE, list_make1(arg), NULL,
@@ -4119,7 +4174,7 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
Node *arg = transformJsonValueExpr(pstate, "JSON_SERIALIZE()",
expr->expr,
JS_FORMAT_JSON,
- InvalidOid);
+ InvalidOid, false);
if (expr->output)
{
@@ -4153,3 +4208,568 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_SERIALIZE, list_make1(arg),
NULL, returning, false, false, expr->location);
}
+
+/*
+ * Transform JSON_VALUE, JSON_QUERY, JSON_EXISTS functions into a JsonExpr node.
+ */
+static Node *
+transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
+{
+ JsonExpr *jsexpr = NULL;
+ const char *func_name = NULL;
+ JsonBehavior *on_error = NULL,
+ *on_empty = NULL;
+
+ /*
+ * Disallow FORMAT specification in the RETURNING clause of JSON_EXISTS()
+ * and JSON_VALUE().
+ */
+ if (func->output &&
+ (func->op == JSON_VALUE_OP || func->op == JSON_EXISTS_OP))
+ {
+ JsonFormat *format = func->output->returning->format;
+
+ if (format->format_type != JS_FORMAT_DEFAULT ||
+ format->encoding != JS_ENC_DEFAULT)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot specify FORMAT in RETURNING clause of %s",
+ func->op == JSON_VALUE_OP ? "JSON_VALUE()" :
+ "JSON_EXISTS()"),
+ parser_errposition(pstate, format->location)));
+ }
+
+ if (func->behavior)
+ {
+ on_empty = linitial(func->behavior);
+ on_error = lsecond(func->behavior);
+ }
+
+ switch (func->op)
+ {
+ case JSON_EXISTS_OP:
+ func_name = "JSON_EXISTS";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ jsexpr->returning->typid = BOOLOID;
+ jsexpr->returning->typmod = -1;
+ }
+ else if (jsexpr->returning->typid != BOOLOID)
+ {
+ Node *coercion_expr;
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+ int location = exprLocation((Node *) jsexpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the
+ * result of evaluating JSON_EXISTS to the coercion
+ * expression.
+ */
+ placeholder->typeId = BOOLOID;
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ coercion_expr =
+ coerce_to_target_type(pstate, (Node *) placeholder, BOOLOID,
+ jsexpr->returning->typid,
+ jsexpr->returning->typmod,
+ COERCION_EXPLICIT,
+ COERCE_IMPLICIT_CAST,
+ location);
+
+ if (coercion_expr == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast type %s to %s",
+ format_type_be(BOOLOID),
+ format_type_be(jsexpr->returning->typid)),
+ parser_coercion_errposition(pstate, location, (Node *) jsexpr)));
+
+ if (coercion_expr != (Node *) placeholder)
+ jsexpr->result_coercion = coercion_expr;
+ }
+
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_FALSE,
+ jsexpr->returning);
+ break;
+
+ case JSON_QUERY_OP:
+ func_name = "JSON_QUERY";
+
+ if (func->wrapper != JSW_NONE && func->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(pstate, func->location)));
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->wrapper = func->wrapper;
+ jsexpr->omit_quotes = (func->quotes == JS_QUOTES_OMIT);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ JsonReturning *ret = jsexpr->returning;
+
+ ret->typid = JsonFuncExprDefaultReturnType(jsexpr);
+ ret->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ if (on_empty)
+ jsexpr->on_empty = transformJsonBehavior(pstate, on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+
+ case JSON_VALUE_OP:
+ func_name = "JSON_VALUE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ /* Make JSON_VALUE return text by default */
+ jsexpr->returning->typid = TEXTOID;
+ jsexpr->returning->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ /*
+ * Initialize expressions to coerce the scalar value returned
+ * by JsonPathValue() to the "returning" type.
+ */
+ if (jsexpr->result_coercion)
+ jsexpr->item_coercions =
+ InitJsonItemCoercions(pstate, jsexpr->returning,
+ exprType(jsexpr->formatted_expr));
+
+ if (on_empty)
+ jsexpr->on_empty = transformJsonBehavior(pstate, on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+ }
+
+ return (Node *) jsexpr;
+}
+
+/*
+ * Common code for JSON_VALUE, JSON_QUERY, JSON_EXISTS transformation
+ * into a JsonExpr node.
+ */
+static JsonExpr *
+transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName)
+{
+ JsonExpr *jsexpr = makeNode(JsonExpr);
+ Node *pathspec;
+
+ jsexpr->location = func->location;
+ jsexpr->op = func->op;
+ jsexpr->formatted_expr = transformJsonValueExpr(pstate, constructName,
+ func->context_item,
+ JS_FORMAT_JSON,
+ InvalidOid, false);
+
+ Assert(jsexpr->formatted_expr != NULL);
+ if (exprType(jsexpr->formatted_expr) != JSONBOID)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("%s() is not yet implemented for the json type",
+ constructName),
+ errhint("Try casting the argument to jsonb"),
+ parser_errposition(pstate, exprLocation(jsexpr->formatted_expr))));
+
+ jsexpr->format = func->context_item->format;
+
+ /* Both set in the caller. */
+ jsexpr->result_coercion = NULL;
+ jsexpr->omit_quotes = false;
+
+ pathspec = transformExprRecurse(pstate, func->pathspec);
+
+ jsexpr->path_spec =
+ coerce_to_target_type(pstate, pathspec, exprType(pathspec),
+ JSONPATHOID, -1,
+ COERCION_EXPLICIT, COERCE_IMPLICIT_CAST,
+ exprLocation(pathspec));
+ if (!jsexpr->path_spec)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("JSON path expression must be of type %s, not of type %s",
+ "jsonpath", format_type_be(exprType(pathspec))),
+ parser_errposition(pstate, exprLocation(pathspec))));
+
+ /*
+ * Transform and coerce to json[b] passing arguments, whose format is
+ * determined by context item type.
+ */
+ transformJsonPassingArgs(pstate, constructName,
+ exprType(jsexpr->formatted_expr) == JSONBOID ?
+ JS_FORMAT_JSONB : JS_FORMAT_JSON,
+ func->passing,
+ &jsexpr->passing_values,
+ &jsexpr->passing_names);
+
+ jsexpr->returning = transformJsonOutput(pstate, func->output, false);
+
+ /* JSON_QUERY supports specifying FORMAT explicitly. */
+ if (func->op != JSON_QUERY_OP)
+ {
+ jsexpr->returning->format->format_type = JS_FORMAT_DEFAULT;
+ jsexpr->returning->format->encoding = JS_ENC_DEFAULT;
+ }
+
+ return jsexpr;
+}
+
+/*
+ * Transform a JSON PASSING clause.
+ */
+static void
+transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names)
+{
+ ListCell *lc;
+
+ *passing_values = NIL;
+ *passing_names = NIL;
+
+ foreach(lc, args)
+ {
+ JsonArgument *arg = castNode(JsonArgument, lfirst(lc));
+ Node *expr = transformJsonValueExpr(pstate, constructName,
+ arg->val, format,
+ InvalidOid, true);
+
+ *passing_values = lappend(*passing_values, expr);
+ *passing_names = lappend(*passing_names, makeString(arg->name));
+ }
+}
+
+/*
+ * Transform the JSON output clause of JSON_VALUE and JSON_QUERY.
+ */
+static Node *
+coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr)
+{
+ JsonReturning *returning = jsexpr->returning;
+ Node *context_item = jsexpr->formatted_expr;
+ int default_typmod;
+ Oid default_typid;
+
+ Assert(returning);
+
+ /*
+ * For JSON_QUERY, use forced coercion via I/O to implement the specified
+ * QUOTES or WRAPPER behavior.
+ */
+ if (jsexpr->op == JSON_QUERY_OP &&
+ (jsexpr->omit_quotes || jsexpr->wrapper != JSW_NONE))
+ {
+ JsonCoercion *coercion = makeNode(JsonCoercion);
+
+ coercion->targettype = returning->typid;
+ coercion->targettypmod = returning->typmod;
+ coercion->via_io = jsexpr->omit_quotes;
+ coercion->via_populate = jsexpr->wrapper != JSW_NONE;
+
+ return (Node *) coercion;
+ }
+
+ default_typid = JsonFuncExprDefaultReturnType(jsexpr);
+ default_typmod = -1;
+ if (returning->typid != default_typid ||
+ returning->typmod != default_typmod)
+ {
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * evaluating the JSON_VALUE/QUERY jsonpath expression to the coercion
+ * function.
+ */
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ placeholder->typeId = exprType(context_item);
+ placeholder->typeMod = exprTypmod(context_item);
+ placeholder->collation = exprCollation(context_item);
+
+ Assert(placeholder->typeId == default_typid);
+ Assert(placeholder->typeMod == default_typmod);
+
+ return coerceJsonExpr(pstate, (Node *) placeholder, returning);
+ }
+
+ return NULL;
+}
+
+/* Returns the default type for a given JsonExpr for a given JsonFormat. */
+static Oid
+JsonFuncExprDefaultReturnType(JsonExpr *jsexpr)
+{
+ JsonFormat *format = jsexpr->format;
+ Node *context_item = jsexpr->formatted_expr;
+
+ Assert(format);
+ if (format->format_type == JS_FORMAT_JSONB)
+ return JSONBOID;
+ else if (format->format_type == JS_FORMAT_DEFAULT &&
+ exprType(context_item) == JSONBOID)
+ return JSONBOID;
+
+ return JSONOID;
+}
+
+/*
+ * Returns a JsonCoercion node to coerce a jsonb-valued expression to the
+ * target type given by 'returning' using either json_populate_type() or
+ * by using the target type's input function.
+ */
+static JsonCoercion *
+makeJsonCoercion(const JsonReturning *returning)
+{
+ JsonCoercion *coercion = makeNode(JsonCoercion);
+ char typtype;
+
+ coercion->targettype = returning->typid;
+ coercion->targettypmod = returning->typmod;
+
+ typtype = get_typtype(returning->typid);
+ if (returning->typid == RECORDOID ||
+ typtype == TYPTYPE_COMPOSITE ||
+ typtype == TYPTYPE_DOMAIN ||
+ type_is_array(returning->typid))
+ coercion->via_populate = true;
+ else
+ coercion->via_io = true;
+
+ return coercion;
+}
+
+/*
+ * Set up a JsonCoercion node to:
+ *
+ * - coerce expression to the output returning type, or
+ * - coerce using json_populate_type() if returning type requires it, or
+ * - coerce via I/O.
+ */
+static Node *
+coerceJsonExpr(ParseState *pstate, Node *expr, const JsonReturning *returning)
+{
+ Node *coerced_expr;
+
+ coerced_expr = coerceJsonFuncExpr(pstate, expr, returning, false);
+ if (coerced_expr)
+ {
+ if (coerced_expr == expr)
+ return NULL;
+ return coerced_expr;
+ }
+
+ return (Node *) makeJsonCoercion(returning);
+}
+
+/*
+ * Initialize JsonCoercion nodes for coercing a given JSON item value produced
+ * by JSON_VALUE to the target "returning" type; also see
+ * ExecPrepareJsonItemCoercion().
+ */
+static List *
+InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId)
+{
+ List *item_coercions = NIL;
+ int i;
+ Oid typeoid;
+ struct
+ {
+ JsonItemType item_type;
+ Oid typeoid;
+ } item_types[] =
+ {
+ {JsonItemTypeNull, UNKNOWNOID},
+ {JsonItemTypeString, TEXTOID},
+ {JsonItemTypeNumeric, NUMERICOID},
+ {JsonItemTypeBoolean, BOOLOID},
+ {JsonItemTypeDate, DATEOID},
+ {JsonItemTypeTime, TIMEOID},
+ {JsonItemTypeTimetz, TIMETZOID},
+ {JsonItemTypeTimestamp, TIMESTAMPOID},
+ {JsonItemTypeTimestamptz, TIMESTAMPTZOID},
+ {JsonItemTypeComposite, contextItemTypeId},
+ {JsonItemTypeInvalid, InvalidOid}
+ };
+
+ for (i = 0; OidIsValid(typeoid = item_types[i].typeoid); i++)
+ {
+ Node *expr;
+ JsonItemCoercion *item_coercion = makeNode(JsonItemCoercion);
+
+ if (typeoid == UNKNOWNOID)
+ {
+ expr = (Node *) makeNullConst(UNKNOWNOID, -1, InvalidOid);
+ }
+ else
+ {
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * JSON_VALUE jsonpath expression to the coercion function.
+ */
+ placeholder->typeId = item_types[i].typeoid;
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ expr = (Node *) placeholder;
+ }
+
+ item_coercion->item_type = item_types[i].item_type;
+ item_coercion->coercion = coerceJsonExpr(pstate, expr, returning);
+ item_coercions = lappend(item_coercions, item_coercion);
+ }
+
+ return item_coercions;
+}
+
+/*
+ * Returns constant values to be returned to the user for various
+ * non-ERROR ON ERROR/EMPTY behaviors.
+ *
+ * Note that JSON_BEHAVIOR_DEFAULT expression is handled by the
+ * caller separately.
+ */
+static Node *
+GetJsonBehaviorConstExpr(JsonBehaviorType btype, int location)
+{
+ Datum val;
+ Oid typid = JSONBOID;
+ int len = -1;
+ bool isnull = false;
+ bool isbyval = false;
+ Const *con;
+
+ switch (btype)
+ {
+ case JSON_BEHAVIOR_EMPTY_ARRAY:
+ val = DirectFunctionCall1(jsonb_in, CStringGetDatum("[]"));
+ break;
+
+ case JSON_BEHAVIOR_EMPTY_OBJECT:
+ val = DirectFunctionCall1(jsonb_in, CStringGetDatum("{}"));
+ break;
+
+ case JSON_BEHAVIOR_TRUE:
+ val = BoolGetDatum(true);
+ /* fall through */
+
+ case JSON_BEHAVIOR_FALSE:
+ val = BoolGetDatum(false);
+ typid = BOOLOID;
+ len = sizeof(bool);
+ isbyval = true;
+ break;
+
+ case JSON_BEHAVIOR_NULL:
+ case JSON_BEHAVIOR_UNKNOWN:
+ case JSON_BEHAVIOR_EMPTY:
+ val = (Datum) 0;
+ isnull = true;
+ typid = INT4OID;
+ len = sizeof(int32);
+ isbyval = true;
+ break;
+
+ case JSON_BEHAVIOR_DEFAULT:
+ case JSON_BEHAVIOR_ERROR:
+ /* Always handled in the caller. */
+ Assert(false);
+ break;
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON behavior %d", btype);
+ break;
+ }
+
+ con = makeConst(typid, -1, InvalidOid, len, val, isnull, isbyval);
+ con->location = location;
+
+ return (Node *) con;
+}
+
+/*
+ * Transform a JSON BEHAVIOR clause.
+ */
+static JsonBehavior *
+transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning)
+{
+ JsonBehaviorType behavior_type = default_behavior;
+ Node *expr = NULL;
+ int location = -1;
+
+ if (behavior)
+ {
+ behavior_type = behavior->btype;
+ location = behavior->location;
+ if (behavior_type == JSON_BEHAVIOR_DEFAULT)
+ expr = transformExprRecurse(pstate, behavior->expr);
+ }
+
+ if (expr == NULL && behavior_type != JSON_BEHAVIOR_ERROR)
+ expr = GetJsonBehaviorConstExpr(behavior_type, location);
+
+ behavior = makeJsonBehavior(behavior_type, expr, location);
+
+ return expr == NULL ? behavior :
+ coerceJsonBehaviorExpr(pstate, behavior, returning);
+}
+
+/*
+ * Coerce expression in `DEFAULT expression ON ERROR / EMPTY` to the target
+ * output type.
+ */
+static JsonBehavior *
+coerceJsonBehaviorExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning)
+{
+ Node *expr = behavior->expr;
+ Node *coerced_expr;
+
+ Assert(expr != NULL);
+
+ coerced_expr =
+ coerce_to_target_type(pstate, expr, exprType(expr),
+ returning->typid, returning->typmod,
+ COERCION_EXPLICIT, COERCE_EXPLICIT_CAST,
+ exprLocation((Node *) behavior));
+
+ if (coerced_expr == NULL)
+ {
+ /* Throw an error right away for EMPTY ARRAY/OBJECT values */
+ if (behavior->btype == JSON_BEHAVIOR_EMPTY_ARRAY ||
+ behavior->btype == JSON_BEHAVIOR_EMPTY_OBJECT)
+ ereport(ERROR,
+ errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast behavior expression of type %s to %s",
+ format_type_be(exprType(expr)),
+ format_type_be(returning->typid)),
+ parser_errposition(pstate, exprLocation(expr)));
+ behavior->coercion = makeJsonCoercion(returning);
+ }
+ else
+ behavior->expr = coerced_expr;
+
+ return behavior;
+}
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 3bc62ac3ba..8f81624f97 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -1989,6 +1989,21 @@ FigureColnameInternal(Node *node, char **name)
/* make JSON_ARRAYAGG act like a regular function */
*name = "json_arrayagg";
return 2;
+ case T_JsonFuncExpr:
+ /* make SQL/JSON functions act like a regular function */
+ switch (((JsonFuncExpr *) node)->op)
+ {
+ case JSON_EXISTS_OP:
+ *name = "json_exists";
+ return 2;
+ case JSON_QUERY_OP:
+ *name = "json_query";
+ return 2;
+ case JSON_VALUE_OP:
+ *name = "json_value";
+ return 2;
+ }
+ break;
default:
break;
}
diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c
index d176723d95..5d3c01f41f 100644
--- a/src/backend/utils/adt/formatting.c
+++ b/src/backend/utils/adt/formatting.c
@@ -4426,6 +4426,50 @@ parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
}
}
+/*
+ * Parses the datetime format string in 'fmt_str' and returns true if it
+ * contains a timezone specifier, false if not.
+ */
+bool
+datetime_format_has_tz(const char *fmt_str)
+{
+ bool incache;
+ int fmt_len = strlen(fmt_str);
+ int result;
+ FormatNode *format;
+
+ if (fmt_len > DCH_CACHE_SIZE)
+ {
+ /*
+ * Allocate new memory if format picture is bigger than static cache
+ * and do not use cache (call parser always)
+ */
+ incache = false;
+
+ format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode));
+
+ parse_format(format, fmt_str, DCH_keywords,
+ DCH_suff, DCH_index, DCH_FLAG, NULL);
+ }
+ else
+ {
+ /*
+ * Use cache buffers
+ */
+ DCHCacheEntry *ent = DCH_cache_fetch(fmt_str, false);
+
+ incache = true;
+ format = ent->format;
+ }
+
+ result = DCH_datetime_type(format);
+
+ if (!incache)
+ pfree(format);
+
+ return result & DCH_ZONED;
+}
+
/*
* do_to_timestamp: shared code for to_timestamp and to_date
*
diff --git a/src/backend/utils/adt/jsonb.c b/src/backend/utils/adt/jsonb.c
index 6f445f5c2b..e5dca46b96 100644
--- a/src/backend/utils/adt/jsonb.c
+++ b/src/backend/utils/adt/jsonb.c
@@ -2163,3 +2163,34 @@ jsonb_float8(PG_FUNCTION_ARGS)
PG_RETURN_DATUM(retValue);
}
+
+/*
+ * Convert jsonb to a C-string stripping quotes from scalar strings.
+ */
+char *
+JsonbUnquote(Jsonb *jb)
+{
+ if (JB_ROOT_IS_SCALAR(jb))
+ {
+ JsonbValue v;
+
+ (void) JsonbExtractScalar(&jb->root, &v);
+
+ if (v.type == jbvString)
+ return pnstrdup(v.val.string.val, v.val.string.len);
+ else if (v.type == jbvBool)
+ return pstrdup(v.val.boolean ? "true" : "false");
+ else if (v.type == jbvNumeric)
+ return DatumGetCString(DirectFunctionCall1(numeric_out,
+ PointerGetDatum(v.val.numeric)));
+ else if (v.type == jbvNull)
+ return pstrdup("null");
+ else
+ {
+ elog(ERROR, "unrecognized jsonb value type %d", v.type);
+ return NULL;
+ }
+ }
+ else
+ return JsonbToCString(NULL, &jb->root, VARSIZE(jb));
+}
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 1574ed5985..b495aedce1 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2805,7 +2805,8 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
- if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc) ||
+ JsonContainerIsScalar(jbc))
{
populate_array_report_expected_array(ctx, ndim - 1);
/* Getting here means the error was reported softly. */
@@ -2813,8 +2814,6 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
return false;
}
- Assert(!JsonContainerIsScalar(jbc));
-
it = JsonbIteratorInit(jbc);
tok = JsonbIteratorNext(&it, &val, true);
@@ -3349,6 +3348,53 @@ populate_record_field(ColumnIOData *col,
}
}
+/* recursively populate specified type from a json/jsonb value */
+Datum
+json_populate_type(Datum json_val, Oid json_type, Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext)
+{
+ JsValue jsv = {0};
+ JsonbValue jbv;
+
+ jsv.is_json = json_type == JSONOID;
+
+ if (*isnull)
+ {
+ if (jsv.is_json)
+ jsv.val.json.str = NULL;
+ else
+ jsv.val.jsonb = NULL;
+ }
+ else if (jsv.is_json)
+ {
+ text *json = DatumGetTextPP(json_val);
+
+ jsv.val.json.str = VARDATA_ANY(json);
+ jsv.val.json.len = VARSIZE_ANY_EXHDR(json);
+ jsv.val.json.type = JSON_TOKEN_INVALID; /* not used in
+ * populate_composite() */
+ }
+ else
+ {
+ Jsonb *jsonb = DatumGetJsonbP(json_val);
+
+ jsv.val.jsonb = &jbv;
+
+ /* fill binary jsonb value pointing to jb */
+ jbv.type = jbvBinary;
+ jbv.val.binary.data = &jsonb->root;
+ jbv.val.binary.len = VARSIZE(jsonb) - VARHDRSZ;
+ }
+
+ if (!*cache)
+ *cache = MemoryContextAllocZero(mcxt, sizeof(ColumnIOData));
+
+ return populate_record_field(*cache, typid, typmod, NULL, mcxt,
+ PointerGetDatum(NULL), &jsv, isnull,
+ escontext);
+}
+
static RecordIOData *
allocate_record_info(MemoryContext mcxt, int ncolumns)
{
diff --git a/src/backend/utils/adt/jsonpath.c b/src/backend/utils/adt/jsonpath.c
index c5ba3b7f1d..9d53e4a992 100644
--- a/src/backend/utils/adt/jsonpath.c
+++ b/src/backend/utils/adt/jsonpath.c
@@ -68,7 +68,9 @@
#include "libpq/pqformat.h"
#include "nodes/miscnodes.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/builtins.h"
+#include "utils/formatting.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
@@ -1110,3 +1112,256 @@ jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from, JsonPathItem *to,
return true;
}
+
+/* SQL/JSON datatype status: */
+typedef enum JsonPathDatatypeStatus
+{
+ jpdsNonDateTime, /* null, bool, numeric, string, array, object */
+ jpdsUnknownDateTime, /* unknown datetime type */
+ jpdsDateTimeZoned, /* timetz, timestamptz */
+ jpdsDateTimeNonZoned, /* time, timestamp, date */
+} JsonPathDatatypeStatus;
+
+/* Context for jspIsMutableWalker() */
+typedef struct JsonPathMutableContext
+{
+ List *varnames; /* list of variable names */
+ List *varexprs; /* list of variable expressions */
+ JsonPathDatatypeStatus current; /* status of @ item */
+ bool lax; /* jsonpath is lax or strict */
+ bool mutable; /* resulting mutability status */
+} JsonPathMutableContext;
+
+/*
+ * Recursive walker for jspIsMutable()
+ */
+static JsonPathDatatypeStatus
+jspIsMutableWalker(JsonPathItem *jpi, JsonPathMutableContext *cxt)
+{
+ JsonPathItem next;
+ JsonPathDatatypeStatus status = jpdsNonDateTime;
+
+ while (!cxt->mutable)
+ {
+ JsonPathItem arg;
+ JsonPathDatatypeStatus leftStatus;
+ JsonPathDatatypeStatus rightStatus;
+
+ switch (jpi->type)
+ {
+ case jpiRoot:
+ Assert(status == jpdsNonDateTime);
+ break;
+
+ case jpiCurrent:
+ Assert(status == jpdsNonDateTime);
+ status = cxt->current;
+ break;
+
+ case jpiFilter:
+ {
+ JsonPathDatatypeStatus prevStatus = cxt->current;
+
+ cxt->current = status;
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+
+ cxt->current = prevStatus;
+ break;
+ }
+
+ case jpiVariable:
+ {
+ int32 len;
+ const char *name = jspGetString(jpi, &len);
+ ListCell *lc1;
+ ListCell *lc2;
+
+ Assert(status == jpdsNonDateTime);
+
+ forboth(lc1, cxt->varnames, lc2, cxt->varexprs)
+ {
+ String *varname = lfirst_node(String, lc1);
+ Node *varexpr = lfirst(lc2);
+
+ if (strncmp(varname->sval, name, len))
+ continue;
+
+ switch (exprType(varexpr))
+ {
+ case DATEOID:
+ case TIMEOID:
+ case TIMESTAMPOID:
+ status = jpdsDateTimeNonZoned;
+ break;
+
+ case TIMETZOID:
+ case TIMESTAMPTZOID:
+ status = jpdsDateTimeZoned;
+ break;
+
+ default:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ break;
+ }
+ break;
+ }
+
+ case jpiEqual:
+ case jpiNotEqual:
+ case jpiLess:
+ case jpiGreater:
+ case jpiLessOrEqual:
+ case jpiGreaterOrEqual:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ leftStatus = jspIsMutableWalker(&arg, cxt);
+
+ jspGetRightArg(jpi, &arg);
+ rightStatus = jspIsMutableWalker(&arg, cxt);
+
+ /*
+ * Comparison of datetime type with different timezone status
+ * is mutable.
+ */
+ if (leftStatus != jpdsNonDateTime &&
+ rightStatus != jpdsNonDateTime &&
+ (leftStatus == jpdsUnknownDateTime ||
+ rightStatus == jpdsUnknownDateTime ||
+ leftStatus != rightStatus))
+ cxt->mutable = true;
+ break;
+
+ case jpiNot:
+ case jpiIsUnknown:
+ case jpiExists:
+ case jpiPlus:
+ case jpiMinus:
+ Assert(status == jpdsNonDateTime);
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiAnd:
+ case jpiOr:
+ case jpiAdd:
+ case jpiSub:
+ case jpiMul:
+ case jpiDiv:
+ case jpiMod:
+ case jpiStartsWith:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ jspGetRightArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiIndexArray:
+ for (int i = 0; i < jpi->content.array.nelems; i++)
+ {
+ JsonPathItem from;
+ JsonPathItem to;
+
+ if (jspGetArraySubscript(jpi, &from, &to, i))
+ jspIsMutableWalker(&to, cxt);
+
+ jspIsMutableWalker(&from, cxt);
+ }
+ /* FALLTHROUGH */
+
+ case jpiAnyArray:
+ if (!cxt->lax)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiAny:
+ if (jpi->content.anybounds.first > 0)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiDatetime:
+ if (jpi->content.arg)
+ {
+ char *template;
+
+ jspGetArg(jpi, &arg);
+ if (arg.type != jpiString)
+ {
+ status = jpdsNonDateTime;
+ break; /* there will be runtime error */
+ }
+
+ template = jspGetString(&arg, NULL);
+ if (datetime_format_has_tz(template))
+ status = jpdsDateTimeZoned;
+ else
+ status = jpdsDateTimeNonZoned;
+ }
+ else
+ {
+ status = jpdsUnknownDateTime;
+ }
+ break;
+
+ case jpiLikeRegex:
+ Assert(status == jpdsNonDateTime);
+ jspInitByBuffer(&arg, jpi->base, jpi->content.like_regex.expr);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ /* literals */
+ case jpiNull:
+ case jpiString:
+ case jpiNumeric:
+ case jpiBool:
+ /* accessors */
+ case jpiKey:
+ case jpiAnyKey:
+ /* special items */
+ case jpiSubscript:
+ case jpiLast:
+ /* item methods */
+ case jpiType:
+ case jpiSize:
+ case jpiAbs:
+ case jpiFloor:
+ case jpiCeiling:
+ case jpiDouble:
+ case jpiKeyValue:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ if (!jspGetNext(jpi, &next))
+ break;
+
+ jpi = &next;
+ }
+
+ return status;
+}
+
+/*
+ * Check whether jsonpath expression is immutable or not.
+ */
+bool
+jspIsMutable(JsonPath *path, List *varnames, List *varexprs)
+{
+ JsonPathMutableContext cxt;
+ JsonPathItem jpi;
+
+ cxt.varnames = varnames;
+ cxt.varexprs = varexprs;
+ cxt.current = jpdsNonDateTime;
+ cxt.lax = (path->header & JSONPATH_LAX) != 0;
+ cxt.mutable = false;
+
+ jspInit(&jpi, path);
+ jspIsMutableWalker(&jpi, &cxt);
+
+ return cxt.mutable;
+}
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 2d0599b4aa..6c5602c64d 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -87,12 +87,16 @@ typedef struct JsonBaseObjectInfo
int id;
} JsonBaseObjectInfo;
+typedef int (*JsonPathVarCallback) (void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
+
/*
* Context of jsonpath execution.
*/
typedef struct JsonPathExecContext
{
- Jsonb *vars; /* variables to substitute into jsonpath */
+ void *vars; /* variables to substitute into jsonpath */
+ JsonPathVarCallback getVar;
JsonbValue *root; /* for $ evaluation */
JsonbValue *current; /* for @ evaluation */
JsonBaseObjectInfo baseObject; /* "base object" for .keyvalue()
@@ -174,7 +178,8 @@ typedef JsonPathBool (*JsonPathPredicateCallback) (JsonPathItem *jsp,
void *param);
typedef Numeric (*BinaryArithmFunc) (Numeric num1, Numeric num2, bool *error);
-static JsonPathExecResult executeJsonPath(JsonPath *path, Jsonb *vars,
+static JsonPathExecResult executeJsonPath(JsonPath *path, void *vars,
+ JsonPathVarCallback getVar,
Jsonb *json, bool throwErrors,
JsonValueList *result, bool useTz);
static JsonPathExecResult executeItem(JsonPathExecContext *cxt,
@@ -225,8 +230,13 @@ static JsonPathExecResult appendBoolResult(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonValueList *found, JsonPathBool res);
static void getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
JsonbValue *value);
+static int GetJsonPathVar(void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
static void getJsonPathVariable(JsonPathExecContext *cxt,
- JsonPathItem *variable, Jsonb *vars, JsonbValue *value);
+ JsonPathItem *variable, JsonbValue *value);
+static int getJsonPathVariableFromJsonb(void *varsJsonb, char *varName,
+ int varNameLen, JsonbValue *val,
+ JsonbValue *baseObject);
static int JsonbArraySize(JsonbValue *jb);
static JsonPathBool executeComparison(JsonPathItem *cmp, JsonbValue *lv,
JsonbValue *rv, void *p);
@@ -284,7 +294,8 @@ jsonb_path_exists_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- res = executeJsonPath(jp, vars, jb, !silent, NULL, tz);
+ res = executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, NULL, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -339,7 +350,8 @@ jsonb_path_match_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -417,7 +429,8 @@ jsonb_path_query_internal(FunctionCallInfo fcinfo, bool tz)
vars = PG_GETARG_JSONB_P_COPY(2);
silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
funcctx->user_fctx = JsonValueListGetList(&found);
@@ -464,7 +477,8 @@ jsonb_path_query_array_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_RETURN_JSONB_P(JsonbValueToJsonb(wrapItemsInArray(&found)));
}
@@ -495,7 +509,8 @@ jsonb_path_query_first_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
if (JsonValueListLength(&found) >= 1)
PG_RETURN_JSONB_P(JsonbValueToJsonb(JsonValueListHead(&found)));
@@ -537,8 +552,9 @@ jsonb_path_query_first_tz(PG_FUNCTION_ARGS)
* In other case it tries to find all the satisfied result items.
*/
static JsonPathExecResult
-executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
- JsonValueList *result, bool useTz)
+executeJsonPath(JsonPath *path, void *vars, JsonPathVarCallback getVar,
+ Jsonb *json, bool throwErrors, JsonValueList *result,
+ bool useTz)
{
JsonPathExecContext cxt;
JsonPathExecResult res;
@@ -550,22 +566,16 @@ executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
if (!JsonbExtractScalar(&json->root, &jbv))
JsonbInitBinary(&jbv, json);
- if (vars && !JsonContainerIsObject(&vars->root))
- {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("\"vars\" argument is not an object"),
- errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
- }
-
cxt.vars = vars;
+ cxt.getVar = getVar;
cxt.laxMode = (path->header & JSONPATH_LAX) != 0;
cxt.ignoreStructuralErrors = cxt.laxMode;
cxt.root = &jbv;
cxt.current = &jbv;
cxt.baseObject.jbc = NULL;
cxt.baseObject.id = 0;
- cxt.lastGeneratedObjectId = vars ? 2 : 1;
+ /* 1 + number of base objects in vars */
+ cxt.lastGeneratedObjectId = 1 + getVar(vars, NULL, 0, NULL, NULL);
cxt.innermostArraySize = -1;
cxt.throwErrors = throwErrors;
cxt.useTz = useTz;
@@ -2108,54 +2118,118 @@ getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
&value->val.string.len);
break;
case jpiVariable:
- getJsonPathVariable(cxt, item, cxt->vars, value);
+ getJsonPathVariable(cxt, item, value);
return;
default:
elog(ERROR, "unexpected jsonpath item type");
}
}
+/*
+ * Returns the computed value of a JSON path variable with given name.
+ */
+static int
+GetJsonPathVar(void *cxt, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject)
+{
+ JsonPathVariable *var = NULL;
+ List *vars = cxt;
+ ListCell *lc;
+ int id = 1;
+
+ if (!varName)
+ return list_length(vars);
+
+ foreach(lc, vars)
+ {
+ JsonPathVariable *curvar = lfirst(lc);
+
+ if (!strncmp(curvar->name, varName, varNameLen))
+ {
+ var = curvar;
+ break;
+ }
+
+ id++;
+ }
+
+ if (!var)
+ return -1;
+
+ if (var->isnull)
+ {
+ val->type = jbvNull;
+ return 0;
+ }
+
+ JsonItemFromDatum(var->value, var->typid, var->typmod, val);
+
+ *baseObject = *val;
+ return id;
+}
+
/*
* Get the value of variable passed to jsonpath executor
*/
static void
getJsonPathVariable(JsonPathExecContext *cxt, JsonPathItem *variable,
- Jsonb *vars, JsonbValue *value)
+ JsonbValue *value)
{
char *varName;
int varNameLength;
+ JsonbValue baseObject;
+ int baseObjectId;
+
+ Assert(variable->type == jpiVariable);
+ varName = jspGetString(variable, &varNameLength);
+
+ if (!cxt->vars ||
+ (baseObjectId = cxt->getVar(cxt->vars, varName, varNameLength, value,
+ &baseObject)) < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("could not find jsonpath variable \"%s\"",
+ pnstrdup(varName, varNameLength))));
+
+ if (baseObjectId > 0)
+ setBaseObject(cxt, &baseObject, baseObjectId);
+}
+
+static int
+getJsonPathVariableFromJsonb(void *varsJsonb, char *varName, int varNameLength,
+ JsonbValue *value, JsonbValue *baseObject)
+{
+ Jsonb *vars = varsJsonb;
JsonbValue tmp;
JsonbValue *v;
- if (!vars)
+ if (!varName)
{
- value->type = jbvNull;
- return;
+ if (vars && !JsonContainerIsObject(&vars->root))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"vars\" argument is not an object"),
+ errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
+ }
+
+ return vars ? 1 : 0; /* count of base objects */
}
- Assert(variable->type == jpiVariable);
- varName = jspGetString(variable, &varNameLength);
tmp.type = jbvString;
tmp.val.string.val = varName;
tmp.val.string.len = varNameLength;
v = findJsonbValueFromContainer(&vars->root, JB_FOBJECT, &tmp);
- if (v)
- {
- *value = *v;
- pfree(v);
- }
- else
- {
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_OBJECT),
- errmsg("could not find jsonpath variable \"%s\"",
- pnstrdup(varName, varNameLength))));
- }
+ if (!v)
+ return -1;
- JsonbInitBinary(&tmp, vars);
- setBaseObject(cxt, &tmp, 1);
+ *value = *v;
+ pfree(v);
+
+ JsonbInitBinary(baseObject, vars);
+ return 1;
}
/**************** Support functions for JsonPath execution *****************/
@@ -2812,3 +2886,240 @@ compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
return DatumGetInt32(DirectFunctionCall2(cmpfunc, val1, val2));
}
+
+/* Executor-callable JSON_EXISTS implementation */
+bool
+JsonPathExists(Datum jb, JsonPath *jp, bool *error, List *vars)
+{
+ JsonPathExecResult res = executeJsonPath(jp, vars, GetJsonPathVar,
+ DatumGetJsonbP(jb), !error, NULL,
+ true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ *error = true;
+
+ return res == jperOk;
+}
+
+/* Executor-callable JSON_QUERY implementation */
+Datum
+JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper, bool *empty,
+ bool *error, List *vars)
+{
+ JsonbValue *first;
+ bool wrap;
+ JsonValueList found = {0};
+ JsonPathExecResult res PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ res = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ {
+ *error = true;
+ *empty = false;
+ return (Datum) 0;
+ }
+
+ count = JsonValueListLength(&found);
+
+ first = count > 0 ? JsonValueListHead(&found) : NULL;
+
+ if (!first)
+ wrap = false;
+ else if (wrapper == JSW_NONE)
+ wrap = false;
+ else if (wrapper == JSW_UNCONDITIONAL)
+ wrap = true;
+ else if (wrapper == JSW_CONDITIONAL)
+ wrap = count > 1 ||
+ IsAJsonbScalar(first) ||
+ (first->type == jbvBinary &&
+ JsonContainerIsScalar(first->val.binary.data));
+ else
+ {
+ elog(ERROR, "unrecognized json wrapper %d", wrapper);
+ wrap = false;
+ }
+
+ if (wrap)
+ return JsonbPGetDatum(JsonbValueToJsonb(wrapItemsInArray(&found)));
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return (Datum) 0;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_QUERY should return singleton item without wrapper"),
+ errhint("Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.")));
+ }
+
+ if (first)
+ return JsonbPGetDatum(JsonbValueToJsonb(first));
+
+ *empty = true;
+ return PointerGetDatum(NULL);
+}
+
+/* Executor-callable JSON_VALUE implementation */
+JsonbValue *
+JsonPathValue(Datum jb, JsonPath *jp, bool *empty, bool *error, List *vars)
+{
+ JsonbValue *res;
+ JsonValueList found = {0};
+ JsonPathExecResult jper PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ jper = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(jper));
+
+ if (error && jperIsError(jper))
+ {
+ *error = true;
+ *empty = false;
+ return NULL;
+ }
+
+ count = JsonValueListLength(&found);
+
+ *empty = (count == 0);
+
+ if (*empty)
+ return NULL;
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ res = JsonValueListHead(&found);
+
+ if (res->type == jbvBinary &&
+ JsonContainerIsScalar(res->val.binary.data))
+ JsonbExtractScalar(res->val.binary.data, res);
+
+ if (!IsAJsonbScalar(res))
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_SCALAR_REQUIRED),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ if (res->type == jbvNull)
+ return NULL;
+
+ return res;
+}
+
+static void
+JsonbValueInitNumericDatum(JsonbValue *jbv, Datum num)
+{
+ jbv->type = jbvNumeric;
+ jbv->val.numeric = DatumGetNumeric(num);
+}
+
+void
+JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
+{
+ switch (typid)
+ {
+ case BOOLOID:
+ res->type = jbvBool;
+ res->val.boolean = DatumGetBool(val);
+ break;
+ case NUMERICOID:
+ JsonbValueInitNumericDatum(res, val);
+ break;
+ case INT2OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int2_numeric, val));
+ break;
+ case INT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int4_numeric, val));
+ break;
+ case INT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int8_numeric, val));
+ break;
+ case FLOAT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float4_numeric, val));
+ break;
+ case FLOAT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float8_numeric, val));
+ break;
+ case TEXTOID:
+ case VARCHAROID:
+ res->type = jbvString;
+ res->val.string.val = VARDATA_ANY(val);
+ res->val.string.len = VARSIZE_ANY_EXHDR(val);
+ break;
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ res->type = jbvDatetime;
+ res->val.datetime.value = val;
+ res->val.datetime.typid = typid;
+ res->val.datetime.typmod = typmod;
+ res->val.datetime.tz = 0;
+ break;
+ case JSONBOID:
+ {
+ JsonbValue *jbv = res;
+ Jsonb *jb = DatumGetJsonbP(val);
+
+ if (JsonContainerIsScalar(&jb->root))
+ {
+ bool result PG_USED_FOR_ASSERTS_ONLY;
+
+ result = JsonbExtractScalar(&jb->root, jbv);
+ Assert(result);
+ }
+ else
+ JsonbInitBinary(jbv, jb);
+ break;
+ }
+ case JSONOID:
+ {
+ text *txt = DatumGetTextP(val);
+ char *str = text_to_cstring(txt);
+ Jsonb *jb;
+
+ jb = DatumGetJsonbP(DirectFunctionCall1(jsonb_in,
+ CStringGetDatum(str)));
+ pfree(str);
+
+ JsonItemFromDatum(JsonbPGetDatum(jb), JSONBOID, -1, res);
+ break;
+ }
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
+ }
+}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index ed7f40f053..a1745c89ff 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -476,6 +476,8 @@ static void get_const_expr(Const *constval, deparse_context *context,
int showtype);
static void get_const_collation(Const *constval, deparse_context *context);
static void get_json_format(JsonFormat *format, StringInfo buf);
+static void get_json_returning(JsonReturning *returning, StringInfo buf,
+ bool json_format_by_default);
static void get_json_constructor(JsonConstructorExpr *ctor,
deparse_context *context, bool showimplicit);
static void get_json_constructor_options(JsonConstructorExpr *ctor,
@@ -518,6 +520,8 @@ static char *generate_qualified_type_name(Oid typid);
static text *string_to_text(char *str);
static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
+static void get_json_path_spec(Node *path_spec, deparse_context *context,
+ bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8300,6 +8304,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_WindowFunc:
case T_FuncExpr:
case T_JsonConstructorExpr:
+ case T_JsonExpr:
/* function-like: name(..) or name[..] */
return true;
@@ -8471,6 +8476,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_GroupingFunc: /* own parentheses */
case T_WindowFunc: /* own parentheses */
case T_CaseExpr: /* other separators */
+ case T_JsonExpr: /* own parentheses */
return true;
default:
return false;
@@ -8586,6 +8592,64 @@ get_rule_expr_paren(Node *node, deparse_context *context,
appendStringInfoChar(context->buf, ')');
}
+static void
+get_json_behavior(JsonBehavior *behavior, deparse_context *context,
+ const char *on)
+{
+ /*
+ * The order of array elements must correspond to the order of
+ * JsonBehaviorType members.
+ */
+ const char *behavior_names[] =
+ {
+ " NULL",
+ " ERROR",
+ " EMPTY",
+ " TRUE",
+ " FALSE",
+ " UNKNOWN",
+ " EMPTY ARRAY",
+ " EMPTY OBJECT",
+ " DEFAULT "
+ };
+
+ if ((int) behavior->btype < 0 || behavior->btype >= lengthof(behavior_names))
+ elog(ERROR, "invalid json behavior type: %d", behavior->btype);
+
+ appendStringInfoString(context->buf, behavior_names[behavior->btype]);
+
+ if (behavior->btype == JSON_BEHAVIOR_DEFAULT)
+ get_rule_expr(behavior->expr, context, false);
+
+ appendStringInfo(context->buf, " ON %s", on);
+}
+
+/*
+ * get_json_expr_options
+ *
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ */
+static void
+get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
+ JsonBehaviorType default_behavior)
+{
+ if (jsexpr->op == JSON_QUERY_OP)
+ {
+ if (jsexpr->wrapper == JSW_CONDITIONAL)
+ appendStringInfo(context->buf, " WITH CONDITIONAL WRAPPER");
+ else if (jsexpr->wrapper == JSW_UNCONDITIONAL)
+ appendStringInfo(context->buf, " WITH UNCONDITIONAL WRAPPER");
+
+ if (jsexpr->omit_quotes)
+ appendStringInfo(context->buf, " OMIT QUOTES");
+ }
+
+ if (jsexpr->on_empty && jsexpr->on_empty->btype != default_behavior)
+ get_json_behavior(jsexpr->on_empty, context, "EMPTY");
+
+ if (jsexpr->on_error && jsexpr->on_error->btype != default_behavior)
+ get_json_behavior(jsexpr->on_error, context, "ERROR");
+}
/* ----------
* get_rule_expr - Parse back an expression
@@ -9745,6 +9809,7 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+
case T_JsonValueExpr:
{
JsonValueExpr *jve = (JsonValueExpr *) node;
@@ -9794,6 +9859,64 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ appendStringInfoString(buf, "JSON_EXISTS(");
+ break;
+ case JSON_QUERY_OP:
+ appendStringInfoString(buf, "JSON_QUERY(");
+ break;
+ case JSON_VALUE_OP:
+ appendStringInfoString(buf, "JSON_VALUE(");
+ break;
+ }
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_json_path_spec(jexpr->path_spec, context, showimplicit);
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoString(buf, " PASSING ");
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ get_rule_expr((Node *) lfirst(lc2), context, showimplicit);
+ appendStringInfo(buf, " AS %s",
+ ((String *) lfirst_node(String, lc1))->sval);
+ }
+ }
+
+ if (jexpr->op != JSON_EXISTS_OP ||
+ jexpr->returning->typid != BOOLOID)
+ get_json_returning(jexpr->returning, context->buf,
+ jexpr->op == JSON_QUERY_OP);
+
+ get_json_expr_options(jexpr, context,
+ jexpr->op != JSON_EXISTS_OP ?
+ JSON_BEHAVIOR_NULL :
+ JSON_BEHAVIOR_FALSE);
+
+ appendStringInfoString(buf, ")");
+ }
+ break;
+
case T_List:
{
char *sep;
@@ -9917,6 +10040,7 @@ looks_like_function(Node *node)
case T_MinMaxExpr:
case T_SQLValueFunction:
case T_XmlExpr:
+ case T_JsonExpr:
/* these are all accepted by func_expr_common_subexpr */
return true;
default:
@@ -10786,6 +10910,18 @@ get_const_collation(Const *constval, deparse_context *context)
}
}
+/*
+ * get_json_path_spec - Parse back a JSON path specification
+ */
+static void
+get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit)
+{
+ if (IsA(path_spec, Const))
+ get_const_expr((Const *) path_spec, context, -1);
+ else
+ get_rule_expr(path_spec, context, showimplicit);
+}
+
/*
* get_json_format - Parse back a JsonFormat node
*/
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index c4fd933154..aecb58664b 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -23,6 +23,8 @@ struct ExprEvalStep;
struct SubscriptingRefState;
struct ScalarArrayOpExprHashTable;
struct JsonConstructorExprState;
+struct JsonbValue;
+struct JsonExprState;
/* Bits in ExprState->flags (see also execnodes.h for public flag bits): */
/* expression's interpreter has been initialized */
@@ -240,6 +242,9 @@ typedef enum ExprEvalOp
EEOP_XMLEXPR,
EEOP_JSON_CONSTRUCTOR,
EEOP_IS_JSON,
+ EEOP_JSONEXPR_PATH,
+ EEOP_JSONEXPR_COERCION,
+ EEOP_JSONEXPR_COERCION_FINISH,
EEOP_AGGREF,
EEOP_GROUPING_FUNC,
EEOP_WINDOW_FUNC,
@@ -692,6 +697,17 @@ typedef struct ExprEvalStep
JsonIsPredicate *pred; /* original expression node */
} is_json;
+ /* for EEOP_JSONEXPR_PATH */
+ struct
+ {
+ struct JsonExprState *jsestate;
+ } jsonexpr;
+
+ /* for EEOP_JSONEXPR_COERCION */
+ struct
+ {
+ struct JsonCoercionState *jcstate;
+ } jsonexpr_coercion;
} d;
} ExprEvalStep;
@@ -755,6 +771,118 @@ typedef struct JsonConstructorExprState
int nargs;
} JsonConstructorExprState;
+/*
+ * Information about the state of JsonPath* evaluation.
+ */
+typedef struct JsonExprPostEvalState
+{
+ /* Did JsonPath* evaluation cause an error? */
+ NullableDatum error;
+
+ /* Is the result of JsonPath* evaluation empty? */
+ NullableDatum empty;
+
+ /*
+ * ExecEvalJsonExprPath() will set this to the address of the step to
+ * use to coerce the result of JsonPath* evaluation to the RETURNING type.
+ * Also see the description of possible step addresses that this could be
+ * set to in the definition of JsonExprState.
+ */
+#define FIELDNO_JSONEXPRPOSTEVALSTATE_JUMP_EVAL_COERCION 2
+ int jump_eval_coercion;
+} JsonExprPostEvalState;
+
+/* State for evaluating a JsonExpr, too big to inline */
+typedef struct JsonExprState
+{
+ /* original expression node */
+ JsonExpr *jsexpr;
+
+ /* value/isnull for formatted_expr */
+ NullableDatum formatted_expr;
+
+ /* value/isnull for pathspec */
+ NullableDatum pathspec;
+
+ /* JsonPathVariable entries for passing_values */
+ List *args;
+
+ /*
+ * Per-row result status info populated by ExecEvalJsonExprPath()
+ * and ExecEvalJsonCoercionFinish().
+ */
+ JsonExprPostEvalState post_eval;
+
+ /*
+ * Address of the step that implements the non-ERROR variant of ON ERROR
+ * and ON EMPTY behaviors, to be jumped to when ExecEvalJsonExprPath()
+ * returns false on encountering an error during JsonPath* evaluation
+ * (ON ERROR) or on finding that no matching JSON item was returned (ON
+ * EMPTY). The same steps are also performed on encountering an error
+ * when coercing JsonPath* result to the RETURNING type.
+ */
+ int jump_error;
+
+ /*
+ * Addresses of steps to perform the coercion of the JsonPath* result value
+ * to the RETURNING type. Each address points to either 1) a special
+ * EEOP_JSONEXPR_COERCION step that handles coercion using the RETURNING
+ * type's input function or by using json_via_populate(), or 2) an
+ * expression such as CoerceViaIO. It may be -1 if no coercion is
+ * necessary.
+ *
+ * jump_eval_result_coercion points to the step to evaluate the coercion
+ * given in JsonExpr.result_coercion.
+ */
+ int jump_eval_result_coercion;
+
+ /* eval_item_coercion_jumps is an array of num_item_coercions elements
+ * each containing a step address to evaluate the coercion from a value of
+ * the given JsonItemType to the RETURNING type, or -1 if no coercion is
+ * necessary. item_coercion_via_expr is an array of boolean flags of the
+ * same length that indicates whether each valid step address in the
+ * eval_item_coercion_jumps array points to an expression or a
+ * EEOP_JSONEXPR_COERCION step. ExecEvalJsonExprPath() will cause an
+ * error if it's the latter, because that mode of coercion is not
+ * supported for all JsonItemTypes.
+ */
+ int num_item_coercions;
+ int *eval_item_coercion_jumps;
+ bool *item_coercion_via_expr;
+
+ /*
+ * For passing when initializing a EEOP_IOCOERCE_SAFE step for any
+ * CoerceViaIO nodes in the expression that must be evaluated in an
+ * error-safe manner.
+ */
+ ErrorSaveContext escontext;
+} JsonExprState;
+
+/*
+ * State for coercing a value to the target type specified in 'coercion' using
+ * either json_populate_type() or by calling the type's input function.
+ */
+typedef struct JsonCoercionState
+{
+ /* original expression node */
+ JsonCoercion *coercion;
+
+ /* Input function info for the target type. */
+ struct
+ {
+ FmgrInfo *finfo;
+ Oid typioparam;
+ } input;
+
+ /* Cache for json_populate_type() */
+ void *cache;
+
+ /*
+ * For soft-error handling in json_populate_type() or
+ * in InputFunctionCallSafe().
+ */
+ ErrorSaveContext *escontext;
+} JsonCoercionState;
/* functions in execExpr.c */
extern void ExprEvalPushStep(ExprState *es, const ExprEvalStep *s);
@@ -809,6 +937,11 @@ extern void ExecEvalXmlExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalJsonConstructor(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op);
+extern void ExecEvalJsonCoercion(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
+extern void ExecEvalJsonCoercionFinish(ExprState *state, ExprEvalStep *op);
+extern bool ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
extern void ExecEvalGroupingFunc(ExprState *state, ExprEvalStep *op);
extern void ExecEvalSubPlan(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
diff --git a/src/include/fmgr.h b/src/include/fmgr.h
index edf61e53f3..8a6b126de1 100644
--- a/src/include/fmgr.h
+++ b/src/include/fmgr.h
@@ -85,6 +85,7 @@ typedef struct FmgrInfo
typedef struct FunctionCallInfoBaseData
{
FmgrInfo *flinfo; /* ptr to lookup info used for this call */
+#define FIELDNO_FUNCTIONCALLINFODATA_CONTEXT 1
fmNodePtr context; /* pass info about context of call */
fmNodePtr resultinfo; /* pass or return extra info about result */
Oid fncollation; /* collation for function to use */
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 3ab86de3ac..de9d8d36c5 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -85,6 +85,7 @@ extern PGDLLIMPORT LLVMTypeRef StructAggState;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerTransData;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerGroupData;
extern PGDLLIMPORT LLVMTypeRef StructPlanState;
+extern PGDLLIMPORT LLVMTypeRef StructJsonExprPostEvalState;
extern PGDLLIMPORT LLVMValueRef AttributeTemplate;
extern PGDLLIMPORT LLVMValueRef ExecEvalBoolSubroutineTemplate;
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 3180703005..a850a1928b 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -112,6 +112,7 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
int location);
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
+extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index e494309da8..f0560c3737 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1692,6 +1692,23 @@ typedef struct TriggerTransition
/* Nodes for SQL/JSON support */
+/*
+ * JsonQuotes -
+ * representation of [KEEP|OMIT] QUOTES clause for JSON_QUERY()
+ */
+typedef enum JsonQuotes
+{
+ JS_QUOTES_UNSPEC, /* unspecified */
+ JS_QUOTES_KEEP, /* KEEP QUOTES */
+ JS_QUOTES_OMIT, /* OMIT QUOTES */
+} JsonQuotes;
+
+/*
+ * JsonPathSpec -
+ * representation of JSON path constant
+ */
+typedef char *JsonPathSpec;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1703,6 +1720,36 @@ typedef struct JsonOutput
JsonReturning *returning; /* RETURNING FORMAT clause and type Oids */
} JsonOutput;
+/*
+ * JsonArgument -
+ * representation of argument from JSON PASSING clause
+ */
+typedef struct JsonArgument
+{
+ NodeTag type;
+ JsonValueExpr *val; /* argument value expression */
+ char *name; /* argument name */
+} JsonArgument;
+
+/*
+ * JsonFuncExpr -
+ * untransformed representation of JSON function expressions
+ */
+typedef struct JsonFuncExpr
+{
+ NodeTag type;
+ JsonExprOp op; /* expression type */
+ JsonValueExpr *context_item; /* context item expression */
+ Node *pathspec; /* JSON path specification expression */
+ List *passing; /* list of PASSING clause arguments, if any */
+ JsonOutput *output; /* output clause, if specified */
+ List *behavior; /* ON EMPTY / ERROR behaviors, list of two
+ * JsonBehavior nodes in that order */
+ JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
+ JsonQuotes quotes; /* omit or keep quotes? (JSON_QUERY only) */
+ int location; /* token location, or -1 if unknown */
+} JsonFuncExpr;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb930afb52..7504b09ca8 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -1552,6 +1552,17 @@ typedef struct XmlExpr
int location;
} XmlExpr;
+/*
+ * JsonExprOp -
+ * enumeration of JSON functions using JSON path
+ */
+typedef enum JsonExprOp
+{
+ JSON_VALUE_OP, /* JSON_VALUE() */
+ JSON_QUERY_OP, /* JSON_QUERY() */
+ JSON_EXISTS_OP, /* JSON_EXISTS() */
+} JsonExprOp;
+
/*
* JsonEncoding -
* representation of JSON ENCODING clause
@@ -1576,6 +1587,37 @@ typedef enum JsonFormatType
* jsonb */
} JsonFormatType;
+/*
+ * JsonBehaviorType -
+ * enumeration of behavior types used in JSON ON ... BEHAVIOR clause
+ *
+ * If enum members are reordered, get_json_behavior() from ruleutils.c
+ * must be updated accordingly.
+ */
+typedef enum JsonBehaviorType
+{
+ JSON_BEHAVIOR_NULL = 0,
+ JSON_BEHAVIOR_ERROR,
+ JSON_BEHAVIOR_EMPTY,
+ JSON_BEHAVIOR_TRUE,
+ JSON_BEHAVIOR_FALSE,
+ JSON_BEHAVIOR_UNKNOWN,
+ JSON_BEHAVIOR_EMPTY_ARRAY,
+ JSON_BEHAVIOR_EMPTY_OBJECT,
+ JSON_BEHAVIOR_DEFAULT,
+} JsonBehaviorType;
+
+/*
+ * JsonWrapper -
+ * representation of WRAPPER clause for JSON_QUERY()
+ */
+typedef enum JsonWrapper
+{
+ JSW_NONE,
+ JSW_CONDITIONAL,
+ JSW_UNCONDITIONAL,
+} JsonWrapper;
+
/*
* JsonFormat -
* representation of JSON FORMAT clause
@@ -1670,6 +1712,94 @@ typedef struct JsonIsPredicate
int location; /* token location, or -1 if unknown */
} JsonIsPredicate;
+/*
+ * JsonItemType
+ * Represents type codes to identify a JsonCoercion node to use when
+ * coercing a given SQL/JSON items to the output SQL type
+ *
+ * The comment next to each item type mentions the JsonbValue.jbvType of the
+ * source JsonbValue value to be coerced using the expression in the
+ * JsonCoercion node.
+ *
+ * Also, see InitJsonItemCoercions() and ExecPrepareJsonItemCoercion().
+ */
+typedef enum JsonItemType
+{
+ JsonItemTypeNull = 0, /* jbvNull */
+ JsonItemTypeString = 1, /* jbvString */
+ JsonItemTypeNumeric = 2, /* jbvNumeric */
+ JsonItemTypeBoolean = 3, /* jbvBool */
+ JsonItemTypeDate = 4, /* jbvDatetime: DATEOID */
+ JsonItemTypeTime = 5, /* jbvDatetime: TIMEOID */
+ JsonItemTypeTimetz = 6, /* jbvDatetime: TIMETZOID */
+ JsonItemTypeTimestamp = 7, /* jbvDatetime: TIMESTAMPOID */
+ JsonItemTypeTimestamptz = 8, /* jbvDatetime: TIMESTAMPTZOID */
+ JsonItemTypeComposite = 9, /* jbvArray, jbvObject, jbvBinary */
+ JsonItemTypeInvalid = 10,
+} JsonItemType;
+
+/*
+ * JsonCoercion -
+ * coercion from SQL/JSON item types to SQL types
+ */
+typedef struct JsonCoercion
+{
+ NodeTag type;
+
+ Oid targettype;
+ int32 targettypmod;
+ bool via_populate; /* coerce result using json_populate_type()? */
+ bool via_io; /* coerce result using type input function? */
+ Oid collation; /* collation for coercion via I/O or populate */
+} JsonCoercion;
+
+typedef struct JsonItemCoercion
+{
+ NodeTag type;
+
+ JsonItemType item_type;
+ Node *coercion;
+} JsonItemCoercion;
+
+/*
+ * JsonBehavior -
+ * representation of a given JSON behavior
+ */
+typedef struct JsonBehavior
+{
+ NodeTag type;
+
+ JsonBehaviorType btype; /* behavior type */
+ Node *expr; /* behavior expression */
+ JsonCoercion *coercion; /* to coerce behavior expression when there is
+ * no cast to the target type */
+ int location; /* token location, or -1 if unknown */
+} JsonBehavior;
+
+/*
+ * JsonExpr -
+ * transformed representation of JSON_VALUE(), JSON_QUERY(), JSON_EXISTS()
+ */
+typedef struct JsonExpr
+{
+ Expr xpr;
+
+ JsonExprOp op; /* json function ID */
+ Node *formatted_expr; /* formatted context item expression */
+ Node *result_coercion; /* resulting coercion to RETURNING type */
+ JsonFormat *format; /* context item format (JSON/JSONB) */
+ Node *path_spec; /* JSON path specification expression */
+ List *passing_names; /* PASSING argument names */
+ List *passing_values; /* PASSING argument values */
+ JsonReturning *returning; /* RETURNING clause type/format info */
+ JsonBehavior *on_empty; /* ON EMPTY behavior */
+ JsonBehavior *on_error; /* ON ERROR behavior */
+ List *item_coercions; /* coercions for JSON_VALUE */
+ JsonWrapper wrapper; /* WRAPPER for JSON_QUERY */
+ bool omit_quotes; /* KEEP/OMIT QUOTES for JSON_QUERY */
+ int location; /* token location, or -1 if unknown */
+} JsonExpr;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..0954d9fc7b 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -93,6 +93,7 @@ PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("conditional", CONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -147,11 +148,13 @@ PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("else", ELSE, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("empty", EMPTY_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("end", END_P, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("error", ERROR_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD, AS_LABEL)
@@ -233,10 +236,14 @@ PG_KEYWORD("join", JOIN, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json", JSON, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_array", JSON_ARRAY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_arrayagg", JSON_ARRAYAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_exists", JSON_EXISTS, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_object", JSON_OBJECT, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("keys", KEYS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -302,6 +309,7 @@ PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("omit", OMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("on", ON, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("only", ONLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -344,6 +352,7 @@ PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("quotes", QUOTES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("read", READ, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("real", REAL, COL_NAME_KEYWORD, BARE_LABEL)
@@ -414,6 +423,7 @@ PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
@@ -449,6 +459,7 @@ PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unbounded", UNBOUNDED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("unconditional", UNCONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("union", UNION, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("unique", UNIQUE, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/utils/formatting.h b/src/include/utils/formatting.h
index 0cad3a2709..9a9ae9754e 100644
--- a/src/include/utils/formatting.h
+++ b/src/include/utils/formatting.h
@@ -29,5 +29,6 @@ extern char *asc_initcap(const char *buff, size_t nbytes);
extern Datum parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
Oid *typid, int32 *typmod, int *tz,
struct Node *escontext);
+extern bool datetime_format_has_tz(const char *fmt_str);
#endif
diff --git a/src/include/utils/jsonb.h b/src/include/utils/jsonb.h
index addc9b608e..613d5953f2 100644
--- a/src/include/utils/jsonb.h
+++ b/src/include/utils/jsonb.h
@@ -422,6 +422,7 @@ extern char *JsonbToCString(StringInfo out, JsonbContainer *in,
int estimated_len);
extern char *JsonbToCStringIndent(StringInfo out, JsonbContainer *in,
int estimated_len);
+extern char *JsonbUnquote(Jsonb *jb);
extern bool JsonbExtractScalar(JsonbContainer *jbc, JsonbValue *res);
extern const char *JsonbTypeName(JsonbValue *val);
diff --git a/src/include/utils/jsonfuncs.h b/src/include/utils/jsonfuncs.h
index 1c6d2be025..4c41eb5540 100644
--- a/src/include/utils/jsonfuncs.h
+++ b/src/include/utils/jsonfuncs.h
@@ -15,6 +15,7 @@
#define JSONFUNCS_H
#include "common/jsonapi.h"
+#include "nodes/nodes.h"
#include "utils/jsonb.h"
/*
@@ -87,5 +88,9 @@ extern Datum datum_to_json(Datum val, JsonTypeCategory tcategory,
extern Datum datum_to_jsonb(Datum val, JsonTypeCategory tcategory,
Oid outfuncoid);
extern Datum jsonb_from_text(text *js, bool unique_keys);
+extern Datum json_populate_type(Datum json_val, Oid json_type,
+ Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext);
#endif
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index f0181e045f..5a37133847 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -16,7 +16,9 @@
#include "fmgr.h"
#include "nodes/pg_list.h"
+#include "nodes/primnodes.h"
#include "utils/jsonb.h"
+#include "utils/jsonfuncs.h"
typedef struct
{
@@ -184,6 +186,7 @@ extern bool jspGetBool(JsonPathItem *v);
extern char *jspGetString(JsonPathItem *v, int32 *len);
extern bool jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from,
JsonPathItem *to, int i);
+extern bool jspIsMutable(JsonPath *path, List *varnames, List *varexprs);
extern const char *jspOperationName(JsonPathItemType type);
@@ -261,4 +264,28 @@ extern bool jspConvertRegexFlags(uint32 xflags, int *result,
struct Node *escontext);
+/*
+ * Evaluation of jsonpath
+ */
+
+/* External variable passed into jsonpath. */
+typedef struct JsonPathVariable
+{
+ char *name;
+ Oid typid;
+ int32 typmod;
+ Datum value;
+ bool isnull;
+} JsonPathVariable;
+
+/* SQL/JSON item */
+extern void JsonItemFromDatum(Datum val, Oid typid, int32 typmod,
+ JsonbValue *res);
+
+extern bool JsonPathExists(Datum jb, JsonPath *path, bool *error, List *vars);
+extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
+ bool *empty, bool *error, List *vars);
+extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
+ bool *error, List *vars);
+
#endif
diff --git a/src/interfaces/ecpg/preproc/ecpg.trailer b/src/interfaces/ecpg/preproc/ecpg.trailer
index 435c139ec2..b2aa44f36d 100644
--- a/src/interfaces/ecpg/preproc/ecpg.trailer
+++ b/src/interfaces/ecpg/preproc/ecpg.trailer
@@ -651,6 +651,34 @@ var_type: simple_type
$$.type_index = mm_strdup("-1");
$$.type_sizeof = NULL;
}
+ | STRING_P
+ {
+ if (INFORMIX_MODE)
+ {
+ /* In Informix mode, "string" is automatically a typedef */
+ $$.type_enum = ECPGt_string;
+ $$.type_str = mm_strdup("char");
+ $$.type_dimension = mm_strdup("-1");
+ $$.type_index = mm_strdup("-1");
+ $$.type_sizeof = NULL;
+ }
+ else
+ {
+ /* Otherwise, legal only if user typedef'ed it */
+ struct typedefs *this = get_typedef("string", false);
+
+ $$.type_str = (this->type->type_enum == ECPGt_varchar || this->type->type_enum == ECPGt_bytea) ? EMPTY : mm_strdup(this->name);
+ $$.type_enum = this->type->type_enum;
+ $$.type_dimension = this->type->type_dimension;
+ $$.type_index = this->type->type_index;
+ if (this->type->type_sizeof && strlen(this->type->type_sizeof) != 0)
+ $$.type_sizeof = this->type->type_sizeof;
+ else
+ $$.type_sizeof = cat_str(3, mm_strdup("sizeof("), mm_strdup(this->name), mm_strdup(")"));
+
+ struct_member_list[struct_level] = ECPGstruct_member_dup(this->struct_member_list);
+ }
+ }
| INTERVAL ecpg_interval
{
$$.type_enum = ECPGt_interval;
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
new file mode 100644
index 0000000000..04d5cc74e3
--- /dev/null
+++ b/src/test/regress/expected/json_sqljson.out
@@ -0,0 +1,18 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ERROR: JSON_EXISTS() is not yet implemented for the json type
+LINE 1: SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ERROR: JSON_VALUE() is not yet implemented for the json type
+LINE 1: SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_QUERY
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ERROR: JSON_QUERY() is not yet implemented for the json type
+LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
new file mode 100644
index 0000000000..cb727e930a
--- /dev/null
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -0,0 +1,1032 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+ json_exists
+-------------
+
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb 'null', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+ json_exists
+-------------
+ f
+(1 row)
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+ json_exists
+-------------
+ t
+(1 row)
+
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL::jsonb, '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+ json_value
+------------
+ true
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+ json_value
+------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+ json_value
+------------
+ 123
+(1 row)
+
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+ERROR: SQL/JSON item cannot be cast to target type
+SELECT JSON_VALUE(jsonb '1.23', '$');
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "1.23"
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+ json_value
+------------
+ aa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "aaa"
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+ json_value
+------------
+ 111
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+ ?column?
+------------
+ 03-01-2017
+(1 row)
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON EMPTY ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb ERROR ON ERROR);
+ERROR: value for domain rgb violates check constraint "rgb_check"
+SELECT JSON_VALUE(jsonb '[]', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '{}', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '1', '$.a');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+ json_value
+------------
+ error
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+ json_value
+------------
+ 0
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: " "
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 5
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+ERROR: cannot specify FORMAT in RETURNING clause of JSON_VALUE()
+LINE 1: ...CT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSO...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+ x | y
+---+----
+ 0 | -2
+ 1 | 2
+ 2 | -1
+(3 rows)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point ERROR ON ERROR);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+-- Test PASSING and RETURNING date/time types
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+ json_value
+--------------------------
+ Tue Feb 20 18:34:56 2018
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING date '2018-02-21 12:34:56 +10' AS ts RETURNING date);
+ json_value
+------------
+ 02-21-2018
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING time '2018-02-21 12:34:56 +10' AS ts RETURNING time);
+ json_value
+------------
+ 12:34:56
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timetz '2018-02-21 12:34:56 +10' AS ts RETURNING timetz);
+ json_value
+-------------
+ 12:34:56+10
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamp '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+ json_value
+--------------------------
+ Wed Feb 21 12:34:56 2018
+(1 row)
+
+-- Also test RETURNING json[b]
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- JSON_QUERY
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+ json_query | json_query | json_query | json_query | json_query
+--------------------+--------------------+--------------------+----------------------+----------------------
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, null, "2"] | [1, null, "2"] | [1, null, "2"] | [[1, null, "2"]] | [[1, null, "2"]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+(6 rows)
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+ unspec | without | with cond | with uncond | with
+--------------------+--------------------+---------------------+----------------------+----------------------
+ | | | |
+ | | | |
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, 2, 3] | [1, 2, 3] | [1, 2, 3] | [[1, 2, 3]] | [[1, 2, 3]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+ | | [1, "2", null, [3]] | [1, "2", null, [3]] | [1, "2", null, [3]]
+(9 rows)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+ json_query
+------------
+ \x616161
+(1 row)
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER ...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER ...
+ ^
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_QUERY should return singleton item without wrapper
+HINT: Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+ json_query
+------------
+ [1,
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+ERROR: cannot cast behavior expression of type jsonb to bytea
+LINE 1: ... JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJE...
+ ^
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+ERROR: cannot cast behavior expression of type jsonb to bytea
+LINE 1: ...jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJE...
+ ^
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+ERROR: cannot cast behavior expression of type jsonb to bigint[]
+LINE 1: ...ON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJE...
+ ^
+SELECT JSON_QUERY(jsonb '"[3,4]"', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+ERROR: cannot cast behavior expression of type jsonb to bigint[]
+LINE 1: ..._QUERY(jsonb '"[3,4]"', '$[*]' RETURNING bigint[] EMPTY OBJE...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+ x | y | list
+---+---+--------------
+ 0 | 0 | []
+ 0 | 1 | [1]
+ 0 | 2 | [1, 2]
+ 0 | 3 | [1, 2, 3]
+ 0 | 4 | [1, 2, 3, 4]
+ 1 | 0 | []
+ 1 | 1 | [1]
+ 1 | 2 | [1, 2]
+ 1 | 3 | [1, 2, 3]
+ 1 | 4 | [1, 2, 3, 4]
+ 2 | 0 | []
+ 2 | 1 | []
+ 2 | 2 | [2]
+ 2 | 3 | [2, 3]
+ 2 | 4 | [2, 3, 4]
+ 3 | 0 | []
+ 3 | 1 | []
+ 3 | 2 | []
+ 3 | 3 | [3]
+ 3 | 4 | [3, 4]
+ 4 | 0 | []
+ 4 | 1 | []
+ 4 | 2 | []
+ 4 | 3 | []
+ 4 | 4 | [4]
+(25 rows)
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+-----------------------------------------------------
+ (1,aaa,"[1, ""2"", {}]","{""x"": [1, ""2"", {}]}",)
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+ unnest
+------------------------
+ {"a": 1, "b": ["foo"]}
+ {"a": 2, "c": {}}
+ 123
+(3 rows)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath ERROR ON ERROR);
+ERROR: syntax error at or near "{" of jsonpath input
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+--------------
+ {1,2,NULL,3}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+ json_query
+------------
+ 1
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- Test constraints
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT '12' ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+);
+\d test_jsonb_constraints
+ Table "public.test_jsonb_constraints"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+--------------------------------------------------------------------------------
+ js | text | | |
+ i | integer | | |
+ x | jsonb | | | JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+Check constraints:
+ "test_jsonb_constraint1" CHECK (js IS JSON)
+ "test_jsonb_constraint2" CHECK (JSON_EXISTS(js::jsonb, '$."a"' PASSING i + 5 AS int, i::text AS txt, ARRAY[1, 2, 3] AS arr))
+ "test_jsonb_constraint3" CHECK (JSON_VALUE(js::jsonb, '$."a"' RETURNING integer DEFAULT 12 ON EMPTY ERROR ON ERROR) > i)
+ "test_jsonb_constraint4" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ "test_jsonb_constraint5" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+ check_clause
+------------------------------------------------------------------------------------------------------------------------
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ (JSON_VALUE((js)::jsonb, '$."a"' RETURNING integer DEFAULT 12 ON EMPTY ERROR ON ERROR) > i)
+ (js IS JSON)
+ JSON_EXISTS((js)::jsonb, '$."a"' PASSING (i + 5) AS int, (i)::text AS txt, ARRAY[1, 2, 3] AS arr)
+(5 rows)
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint1"
+DETAIL: Failing row contains (, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains (1, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ([], null, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ({"b": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint3"
+DETAIL: Failing row contains ({"a": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint5"
+DETAIL: Failing row contains ({"a": 7}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint4"
+DETAIL: Failing row contains ({"a": 10}, 1, [1, 2]).
+DROP TABLE test_jsonb_constraints;
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index f0987ff537..864bf04fe7 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -103,7 +103,7 @@ test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combo
# ----------
# Another group of parallel tests (JSON related)
# ----------
-test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson
+test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson json_sqljson jsonb_sqljson
# ----------
# Another group of parallel tests
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
new file mode 100644
index 0000000000..4f30fa46b9
--- /dev/null
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -0,0 +1,11 @@
+-- JSON_EXISTS
+
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+
+-- JSON_QUERY
+
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
new file mode 100644
index 0000000000..77bfb6d61b
--- /dev/null
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -0,0 +1,337 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+SELECT JSON_EXISTS(jsonb 'null', '$');
+SELECT JSON_EXISTS(jsonb '[]', '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL::jsonb, '$');
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+
+SELECT JSON_VALUE(jsonb '123', '$');
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1.23', '$');
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null);
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON EMPTY ERROR ON ERROR);
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '[]', '$');
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '{}', '$');
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1', '$.a');
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point ERROR ON ERROR);
+
+-- Test PASSING and RETURNING date/time types
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING date '2018-02-21 12:34:56 +10' AS ts RETURNING date);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING time '2018-02-21 12:34:56 +10' AS ts RETURNING time);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timetz '2018-02-21 12:34:56 +10' AS ts RETURNING timetz);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamp '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+
+-- Also test RETURNING json[b]
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- JSON_QUERY
+
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '"[3,4]"', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath);
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath ERROR ON ERROR);
+
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- Test constraints
+
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT '12' ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+);
+
+\d test_jsonb_constraints
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+
+DROP TABLE test_jsonb_constraints;
+
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d659adbfd6..9ee58ec2b0 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1247,6 +1247,7 @@ Join
JoinCostWorkspace
JoinDomain
JoinExpr
+JsonFuncExpr
JoinHashEntry
JoinPath
JoinPathExtraData
@@ -1257,18 +1258,29 @@ JsObject
JsValue
JsonAggConstructor
JsonAggState
+JsonArgument
JsonArrayAgg
JsonArrayConstructor
JsonArrayQueryConstructor
JsonBaseObjectInfo
+JsonBehavior
+JsonBehaviorType
+JsonCoercion
+JsonCoercionState
JsonConstructorExpr
JsonConstructorExprState
JsonConstructorType
JsonEncoding
+JsonExpr
+JsonExprOp
+JsonExprPostEvalState
+JsonExprState
+JsonCoercionState
JsonFormat
JsonFormatType
JsonHashEntry
JsonIsPredicate
+JsonItemType
JsonIterateStringValuesAction
JsonKeyValue
JsonLexContext
@@ -1286,6 +1298,7 @@ JsonParseContext
JsonParseErrorType
JsonPath
JsonPathBool
+JsonPathDatatypeStatus
JsonPathExecContext
JsonPathExecResult
JsonPathGinAddPathItemFunc
@@ -1298,10 +1311,15 @@ JsonPathGinPathItem
JsonPathItem
JsonPathItemType
JsonPathKeyword
+JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
JsonPathString
+JsonPathVarCallback
+JsonPathVariable
+JsonPathVariableEvalContext
+JsonQuotes
JsonReturning
JsonScalarExpr
JsonSemAction
@@ -1318,6 +1336,7 @@ JsonValueExpr
JsonValueList
JsonValueListIterator
JsonValueType
+JsonWrapper
Jsonb
JsonbAggState
JsonbContainer
--
2.35.3
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
@ 2023-11-24 09:32 ` Amit Langote <[email protected]>
1 sibling, 0 replies; 94+ messages in thread
From: Amit Langote @ 2023-11-24 09:32 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; Erik Rijkers <[email protected]>; pgsql-hackers
On Thu, Nov 23, 2023 at 4:38 AM Andres Freund <[email protected]> wrote:
> On 2023-11-21 12:52:35 +0900, Amit Langote wrote:
> > version gram.o text bytes %change gram.c bytes %change
> >
> > 9.6 534010 - 2108984 -
> > 10 582554 9.09 2258313 7.08
> > 11 584596 0.35 2313475 2.44
> > 12 590957 1.08 2341564 1.21
> > 13 590381 -0.09 2357327 0.67
> > 14 600707 1.74 2428841 3.03
> > 15 633180 5.40 2495364 2.73
> > 16 653464 3.20 2575269 3.20
> > 17-sqljson 672800 2.95 2709422 3.97
> >
> > So if we put SQL/JSON (including JSON_TABLE()) into 17, we end up with a gram.o 2.95% larger than v16, which granted is a somewhat larger bump, though also smaller than with some of recent releases.
>
> I think it's ok to increase the size if it's necessary increases - but I also
> think we've been a bit careless at times, and that that has made the parser
> slower. There's probably also some "infrastructure" work we could do combat
> some of the growth too.
>
> I know I triggered the use of the .c bytes and text size, but it'd probably
> more sensible to look at the size of the important tables generated by bison.
> I think the most relevant defines are:
>
> #define YYLAST 117115
> #define YYNTOKENS 521
> #define YYNNTS 707
> #define YYNRULES 3300
> #define YYNSTATES 6255
> #define YYMAXUTOK 758
>
>
> I think a lot of the reason we end up with such a big "state transition" space
> is that a single addition to e.g. col_name_keyword or unreserved_keyword
> increases the state space substantially, because it adds new transitions to so
> many places. We're in quadratic territory, I think. We might be able to do
> some lexer hackery to avoid that, but not sure.
One thing I noticed when looking at the raw parsing times across
versions is that they improved a bit around v12 and then some in v13:
9.0 0.000060 s
9.6 0.000061 s
10 0.000061 s
11 0.000063 s
12 0.000055 s
13 0.000054 s
15 0.000057 s
16 0.000059 s
I think they might be due to the following commits in v12 and v13 resp.:
commit c64d0cd5ce24a344798534f1bc5827a9199b7a6e
Author: Tom Lane <[email protected]>
Date: Wed Jan 9 19:47:38 2019 -0500
Use perfect hashing, instead of binary search, for keyword lookup.
...
Discussion: https://postgr.es/m/[email protected]
commit 7f380c59f800f7e0fb49f45a6ff7787256851a59
Author: Tom Lane <[email protected]>
Date: Mon Jan 13 15:04:31 2020 -0500
Reduce size of backend scanner's tables.
...
Discussion:
https://postgr.es/m/CACPNZCvaoa3EgVWm5yZhcSTX6RAtaLgniCPcBVOCwm8h3xpWkw@mail.gmail.com
I haven't read the whole discussions there to see if the target(s)
included the metrics you've mentioned though, either directly or
indirectly.
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-11-16 04:53 ` Erik Rijkers <[email protected]>
2023-11-16 04:57 ` Re: remaining sql/json patches Amit Langote <[email protected]>
1 sibling, 1 reply; 94+ messages in thread
From: Erik Rijkers @ 2023-11-16 04:53 UTC (permalink / raw)
To: Amit Langote <[email protected]>; Andres Freund <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andrew Dunstan <[email protected]>; jian he <[email protected]>; pgsql-hackers
Op 11/15/23 om 14:00 schreef Amit Langote:
> Hi,
[..]
> Attached updated patch. The version of 0001 that I posted on Oct 11
> to add the error-safe version of CoerceViaIO contained many
> unnecessary bits that are now removed.
>
> --
> Thanks, Amit Langote
> EDB: http://www.enterprisedb.com
> [v24-0001-Add-soft-error-handling-to-some-expression-nodes.patch]
> [v24-0002-Add-soft-error-handling-to-populate_record_field.patch]
> [v24-0003-SQL-JSON-query-functions.patch]
> [v24-0004-JSON_TABLE.patch]
> [v24-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch]
Hi Amit,
Here is a statement that seems to gobble up all memory and to totally
lock up the machine. No ctrl-C - only a power reset gets me out of that.
It was in one of my tests, so it used to work:
select json_query(
jsonb '"[3,4]"'
, '$[*]' returning bigint[] empty object on error
);
Can you have a look?
Thanks,
Erik
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-16 04:53 ` Re: remaining sql/json patches Erik Rijkers <[email protected]>
@ 2023-11-16 04:57 ` Amit Langote <[email protected]>
2023-11-16 06:53 ` Re: remaining sql/json patches Amit Langote <[email protected]>
0 siblings, 1 reply; 94+ messages in thread
From: Amit Langote @ 2023-11-16 04:57 UTC (permalink / raw)
To: Erik Rijkers <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; pgsql-hackers; jian he <[email protected]>
Hi Erik,
On Thu, Nov 16, 2023 at 13:52 Erik Rijkers <[email protected]> wrote:
> Op 11/15/23 om 14:00 schreef Amit Langote:
> > Hi,
>
> [..]
>
> > Attached updated patch. The version of 0001 that I posted on Oct 11
> > to add the error-safe version of CoerceViaIO contained many
> > unnecessary bits that are now removed.
> >
> > --
> > Thanks, Amit Langote
> > EDB: http://www.enterprisedb.com
>
> > [v24-0001-Add-soft-error-handling-to-some-expression-nodes.patch]
> > [v24-0002-Add-soft-error-handling-to-populate_record_field.patch]
> > [v24-0003-SQL-JSON-query-functions.patch]
> > [v24-0004-JSON_TABLE.patch]
> > [v24-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch]
>
> Hi Amit,
>
> Here is a statement that seems to gobble up all memory and to totally
> lock up the machine. No ctrl-C - only a power reset gets me out of that.
> It was in one of my tests, so it used to work:
>
> select json_query(
> jsonb '"[3,4]"'
> , '$[*]' returning bigint[] empty object on error
> );
>
> Can you have a look?
Wow, will look. Thanks.
>
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-16 04:53 ` Re: remaining sql/json patches Erik Rijkers <[email protected]>
2023-11-16 04:57 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-11-16 06:53 ` Amit Langote <[email protected]>
2023-11-21 07:09 ` Re: remaining sql/json patches Peter Eisentraut <[email protected]>
0 siblings, 1 reply; 94+ messages in thread
From: Amit Langote @ 2023-11-16 06:53 UTC (permalink / raw)
To: Erik Rijkers <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; pgsql-hackers; jian he <[email protected]>
On Thu, Nov 16, 2023 at 1:57 PM Amit Langote <[email protected]> wrote:
> On Thu, Nov 16, 2023 at 13:52 Erik Rijkers <[email protected]> wrote:
>> Op 11/15/23 om 14:00 schreef Amit Langote:
>> > [v24-0001-Add-soft-error-handling-to-some-expression-nodes.patch]
>> > [v24-0002-Add-soft-error-handling-to-populate_record_field.patch]
>> > [v24-0003-SQL-JSON-query-functions.patch]
>> > [v24-0004-JSON_TABLE.patch]
>> > [v24-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch]
>>
>> Hi Amit,
>>
>> Here is a statement that seems to gobble up all memory and to totally
>> lock up the machine. No ctrl-C - only a power reset gets me out of that.
>> It was in one of my tests, so it used to work:
>>
>> select json_query(
>> jsonb '"[3,4]"'
>> , '$[*]' returning bigint[] empty object on error
>> );
>>
>> Can you have a look?
>
> Wow, will look. Thanks.
Should be fixed in the attached. The bug was caused by the recent
redesign of JsonExpr evaluation steps.
Your testing is very much appreciated. Thanks.
--
Thanks, Amit Langote
EDB: http://www.enterprisedb.com
Attachments:
[application/octet-stream] v25-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch (2.3K, ../../CA+HiwqF14cAgG7e0SJ90VXd8OKPzeRLBAn3p=-R8JV7H+M35fA@mail.gmail.com/2-v25-0005-Claim-SQL-standard-compliance-for-SQL-JSON-featu.patch)
download | inline diff:
From c60d98efe31c9d6e9daddd6393d6c35ba5ea2a01 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:18 +0900
Subject: [PATCH v25 5/5] Claim SQL standard compliance for SQL/JSON features
Discussion: https://postgr.es/m/[email protected]
---
src/backend/catalog/sql_features.txt | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt
index 80c40eaf57..9500a80f4d 100644
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -548,15 +548,15 @@ T811 Basic SQL/JSON constructor functions YES
T812 SQL/JSON: JSON_OBJECTAGG YES
T813 SQL/JSON: JSON_ARRAYAGG with ORDER BY YES
T814 Colon in JSON_OBJECT or JSON_OBJECTAGG YES
-T821 Basic SQL/JSON query operators NO
+T821 Basic SQL/JSON query operators YES
T822 SQL/JSON: IS JSON WITH UNIQUE KEYS predicate YES
-T823 SQL/JSON: PASSING clause NO
-T824 JSON_TABLE: specific PLAN clause NO
-T825 SQL/JSON: ON EMPTY and ON ERROR clauses NO
-T826 General value expression in ON ERROR or ON EMPTY clauses NO
-T827 JSON_TABLE: sibling NESTED COLUMNS clauses NO
-T828 JSON_QUERY NO
-T829 JSON_QUERY: array wrapper options NO
+T823 SQL/JSON: PASSING clause YES
+T824 JSON_TABLE: specific PLAN clause YES
+T825 SQL/JSON: ON EMPTY and ON ERROR clauses YES
+T826 General value expression in ON ERROR or ON EMPTY clauses YES
+T827 JSON_TABLE: sibling NESTED COLUMNS clauses YES
+T828 JSON_QUERY YES
+T829 JSON_QUERY: array wrapper options YES
T830 Enforcing unique keys in SQL/JSON constructor functions YES
T831 SQL/JSON path language: strict mode YES
T832 SQL/JSON path language: item method YES
@@ -565,7 +565,7 @@ T834 SQL/JSON path language: wildcard member accessor YES
T835 SQL/JSON path language: filter expressions YES
T836 SQL/JSON path language: starts with predicate YES
T837 SQL/JSON path language: regex_like predicate YES
-T838 JSON_TABLE: PLAN DEFAULT clause NO
+T838 JSON_TABLE: PLAN DEFAULT clause YES
T839 Formatted cast of datetimes to/from character strings NO
T840 Hex integer literals in SQL/JSON path language YES
T851 SQL/JSON: optional keywords for default syntax YES
--
2.35.3
[application/octet-stream] v25-0001-Add-soft-error-handling-to-some-expression-nodes.patch (9.1K, ../../CA+HiwqF14cAgG7e0SJ90VXd8OKPzeRLBAn3p=-R8JV7H+M35fA@mail.gmail.com/3-v25-0001-Add-soft-error-handling-to-some-expression-nodes.patch)
download | inline diff:
From 05ca3f9a8776676d7d113b1f0dcf1f1fb815e41e Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Fri, 29 Sep 2023 13:22:15 +0900
Subject: [PATCH v25 1/5] Add soft error handling to some expression nodes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This adjusts the CoerceViaIO and CoerceToDomain expression evaluation
code to handle errors softly.
For CoerceViaIo, this adds a new ExprEvalStep opcode
EEOP_IOCOERCE_SAFE, which is implemented in new function
ExecEvalCoerceViaIOSafe(). The only difference from EEOP_IOCOERCE's
inline implementation is that the input function receives an
ErrorSaveContext via the function's FunctionCallInfo.context, which
it can use to handle errors softly.
For CoerceToDomain, this simply entails replacing the ereport() in
ExecEvalConstraintCheck() by errsave() using the ErrorSaveContext
passed in the expression's ExprEvalStep.
In both cases, the ErrorSaveContext to be used is passed by setting
ExprState.escontext to point to it before calling ExecInitExprRec()
on the expression tree whose errors are to be suppressed.
Note that no call site of ExecInitExprRec() has been changed in this
commit, so there's no functional change. This is intended for
implementing new SQL/JSON expression nodes in future commits that
will use to it suppress errors that may occur during type coercions.
Reviewed-by: Álvaro Herrera
Reviewed-by: Andres Freund
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/executor/execExpr.c | 8 ++-
src/backend/executor/execExprInterp.c | 72 ++++++++++++++++++++++++++-
src/backend/jit/llvm/llvmjit_expr.c | 6 +++
src/backend/jit/llvm/llvmjit_types.c | 1 +
src/include/executor/execExpr.h | 4 ++
src/include/nodes/execnodes.h | 7 +++
6 files changed, 96 insertions(+), 2 deletions(-)
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 2c62b0c9c8..34bd2102b5 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -1563,7 +1563,10 @@ ExecInitExprRec(Expr *node, ExprState *state,
* We don't check permissions here as a type's input/output
* function are assumed to be executable by everyone.
*/
- scratch.opcode = EEOP_IOCOERCE;
+ if (state->escontext == NULL)
+ scratch.opcode = EEOP_IOCOERCE;
+ else
+ scratch.opcode = EEOP_IOCOERCE_SAFE;
/* lookup the source type's output function */
scratch.d.iocoerce.finfo_out = palloc0(sizeof(FmgrInfo));
@@ -1599,6 +1602,8 @@ ExecInitExprRec(Expr *node, ExprState *state,
fcinfo_in->args[2].value = Int32GetDatum(-1);
fcinfo_in->args[2].isnull = false;
+ fcinfo_in->context = (Node *) state->escontext;
+
ExprEvalPushStep(state, &scratch);
break;
}
@@ -3306,6 +3311,7 @@ ExecInitCoerceToDomain(ExprEvalStep *scratch, CoerceToDomain *ctest,
/* we'll allocate workspace only if needed */
scratch->d.domaincheck.checkvalue = NULL;
scratch->d.domaincheck.checknull = NULL;
+ scratch->d.domaincheck.escontext = state->escontext;
/*
* Evaluate argument - it's fine to directly store it into resv/resnull,
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 24c2b60c62..4e152fdfe3 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -63,6 +63,7 @@
#include "executor/nodeSubplan.h"
#include "funcapi.h"
#include "miscadmin.h"
+#include "nodes/miscnodes.h"
#include "nodes/nodeFuncs.h"
#include "parser/parsetree.h"
#include "pgstat.h"
@@ -452,6 +453,7 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_CASE_TESTVAL,
&&CASE_EEOP_MAKE_READONLY,
&&CASE_EEOP_IOCOERCE,
+ &&CASE_EEOP_IOCOERCE_SAFE,
&&CASE_EEOP_DISTINCT,
&&CASE_EEOP_NOT_DISTINCT,
&&CASE_EEOP_NULLIF,
@@ -1205,6 +1207,12 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_IOCOERCE_SAFE)
+ {
+ ExecEvalCoerceViaIOSafe(state, op);
+ EEO_NEXT();
+ }
+
EEO_CASE(EEOP_DISTINCT)
{
/*
@@ -2510,6 +2518,68 @@ ExecEvalParamExtern(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
errmsg("no value found for parameter %d", paramId)));
}
+/*
+ * Evaluate a CoerceViaIO node in soft-error mode.
+ *
+ * The source value is in op's result variable.
+ */
+void
+ExecEvalCoerceViaIOSafe(ExprState *state, ExprEvalStep *op)
+{
+ char *str;
+
+ /* call output function (similar to OutputFunctionCall) */
+ if (*op->resnull)
+ {
+ /* output functions are not called on nulls */
+ str = NULL;
+ }
+ else
+ {
+ FunctionCallInfo fcinfo_out;
+
+ fcinfo_out = op->d.iocoerce.fcinfo_data_out;
+ fcinfo_out->args[0].value = *op->resvalue;
+ fcinfo_out->args[0].isnull = false;
+
+ fcinfo_out->isnull = false;
+ str = DatumGetCString(FunctionCallInvoke(fcinfo_out));
+
+ /* OutputFunctionCall assumes result isn't null */
+ Assert(!fcinfo_out->isnull);
+ }
+
+ /* call input function (similar to InputFunctionCall) */
+ if (!op->d.iocoerce.finfo_in->fn_strict || str != NULL)
+ {
+ FunctionCallInfo fcinfo_in;
+
+ fcinfo_in = op->d.iocoerce.fcinfo_data_in;
+ fcinfo_in->args[0].value = PointerGetDatum(str);
+ fcinfo_in->args[0].isnull = *op->resnull;
+ /* second and third arguments are already set up */
+
+ /* ErrorSaveContext must be present. */
+ Assert(IsA(fcinfo_in->context, ErrorSaveContext));
+
+ fcinfo_in->isnull = false;
+ *op->resvalue = FunctionCallInvoke(fcinfo_in);
+
+ if (SOFT_ERROR_OCCURRED(fcinfo_in->context))
+ {
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ return;
+ }
+
+ /* Should get null result if and only if str is NULL */
+ if (str == NULL)
+ Assert(*op->resnull);
+ else
+ Assert(!*op->resnull);
+ }
+}
+
/*
* Evaluate a SQLValueFunction expression.
*/
@@ -3745,7 +3815,7 @@ ExecEvalConstraintCheck(ExprState *state, ExprEvalStep *op)
{
if (!*op->d.domaincheck.checknull &&
!DatumGetBool(*op->d.domaincheck.checkvalue))
- ereport(ERROR,
+ errsave((Node *) op->d.domaincheck.escontext,
(errcode(ERRCODE_CHECK_VIOLATION),
errmsg("value for domain %s violates check constraint \"%s\"",
format_type_be(op->d.domaincheck.resulttype),
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index a3a0876bff..81856a9dc7 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1431,6 +1431,12 @@ llvm_compile_expr(ExprState *state)
break;
}
+ case EEOP_IOCOERCE_SAFE:
+ build_EvalXFunc(b, mod, "ExecEvalCoerceViaIOSafe",
+ v_state, op);
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
case EEOP_DISTINCT:
case EEOP_NOT_DISTINCT:
{
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 791902ff1f..3a4be09e50 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -162,6 +162,7 @@ void *referenced_functions[] =
ExecEvalRow,
ExecEvalRowNotNull,
ExecEvalRowNull,
+ ExecEvalCoerceViaIOSafe,
ExecEvalSQLValueFunction,
ExecEvalScalarArrayOp,
ExecEvalHashedScalarArrayOp,
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index 048573c2bc..c4fd933154 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -16,6 +16,7 @@
#include "executor/nodeAgg.h"
#include "nodes/execnodes.h"
+#include "nodes/miscnodes.h"
/* forward references to avoid circularity */
struct ExprEvalStep;
@@ -168,6 +169,7 @@ typedef enum ExprEvalOp
/* evaluate assorted special-purpose expression types */
EEOP_IOCOERCE,
+ EEOP_IOCOERCE_SAFE,
EEOP_DISTINCT,
EEOP_NOT_DISTINCT,
EEOP_NULLIF,
@@ -547,6 +549,7 @@ typedef struct ExprEvalStep
bool *checknull;
/* OID of domain type */
Oid resulttype;
+ ErrorSaveContext *escontext;
} domaincheck;
/* for EEOP_CONVERT_ROWTYPE */
@@ -776,6 +779,7 @@ extern void ExecEvalParamExec(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalParamExtern(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
+extern void ExecEvalCoerceViaIOSafe(ExprState *state, ExprEvalStep *op);
extern void ExecEvalSQLValueFunction(ExprState *state, ExprEvalStep *op);
extern void ExecEvalCurrentOfExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalNextValueExpr(ExprState *state, ExprEvalStep *op);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 5d7f17dee0..6a7118d300 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -34,6 +34,7 @@
#include "fmgr.h"
#include "lib/ilist.h"
#include "lib/pairingheap.h"
+#include "nodes/miscnodes.h"
#include "nodes/params.h"
#include "nodes/plannodes.h"
#include "nodes/tidbitmap.h"
@@ -129,6 +130,12 @@ typedef struct ExprState
Datum *innermost_domainval;
bool *innermost_domainnull;
+
+ /*
+ * For expression nodes that support soft errors. Should be set to NULL
+ * before calling ExecInitExprRec() if the caller wants errors thrown.
+ */
+ ErrorSaveContext *escontext;
} ExprState;
--
2.35.3
[application/octet-stream] v25-0002-Add-soft-error-handling-to-populate_record_field.patch (23.2K, ../../CA+HiwqF14cAgG7e0SJ90VXd8OKPzeRLBAn3p=-R8JV7H+M35fA@mail.gmail.com/4-v25-0002-Add-soft-error-handling-to-populate_record_field.patch)
download | inline diff:
From 88d92a7c03d7f6a345f8c86c8a852a76ab858623 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Fri, 29 Sep 2023 13:41:28 +0900
Subject: [PATCH v25 2/5] Add soft error handling to populate_record_field()
An uncoming patch would like the ability to call it from the
executor for some SQL/JSON expression nodes and ask to suppress any
errors that may occur.
This commit does two things mainly:
* It modifies the various interfaces internal to jsonfuncs.c to pass
the ErrorSaveContext around.
* Make necessary modifications to handle the cases where the
processing is aborted partway through various functions that take
an ErrorSaveContext when a soft error occurs.
Note that the above changes are only intended to suppress errors in
the functions in jsonfuncs.c, but not those in any external functions
that the functions in jsonfuncs.c in turn call, such as those from
arrayfuncs.c. It is assumed that the various populate_* functions
validate the data before passing those to external functions.
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
src/backend/utils/adt/jsonfuncs.c | 280 ++++++++++++++++++++++--------
1 file changed, 210 insertions(+), 70 deletions(-)
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index aa37c401e5..1574ed5985 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -265,6 +265,7 @@ typedef struct PopulateArrayContext
int *dims; /* dimensions */
int *sizes; /* current dimension counters */
int ndims; /* number of dimensions */
+ Node *escontext; /* For soft-error handling */
} PopulateArrayContext;
/* state for populate_array_json() */
@@ -389,7 +390,8 @@ static JsonParseErrorType elements_array_element_end(void *state, bool isnull);
static JsonParseErrorType elements_scalar(void *state, char *token, JsonTokenType tokentype);
/* turn a json object into a hash table */
-static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname);
+static HTAB *get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext);
/* semantic actions for populate_array_json */
static JsonParseErrorType populate_array_object_start(void *_state);
@@ -431,37 +433,42 @@ static Datum populate_record_worker(FunctionCallInfo fcinfo, const char *funcnam
/* helper functions for populate_record[set] */
static HeapTupleHeader populate_record(TupleDesc tupdesc, RecordIOData **record_p,
HeapTupleHeader defaultval, MemoryContext mcxt,
- JsObject *obj);
+ JsObject *obj, Node *escontext);
static void get_record_type_from_argument(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
static void get_record_type_from_query(FunctionCallInfo fcinfo,
const char *funcname,
PopulateRecordCache *cache);
-static void JsValueToJsObject(JsValue *jsv, JsObject *jso);
+static bool JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext);
static Datum populate_composite(CompositeIOData *io, Oid typid,
const char *colname, MemoryContext mcxt,
- HeapTupleHeader defaultval, JsValue *jsv, bool isnull);
-static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv);
+ HeapTupleHeader defaultval, JsValue *jsv, bool *isnull,
+ Node *escontext);
+static Datum populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext);
static void prepare_column_cache(ColumnIOData *column, Oid typid, int32 typmod,
MemoryContext mcxt, bool need_scalar);
static Datum populate_record_field(ColumnIOData *col, Oid typid, int32 typmod,
const char *colname, MemoryContext mcxt, Datum defaultval,
- JsValue *jsv, bool *isnull);
+ JsValue *jsv, bool *isnull, Node *escontext);
static RecordIOData *allocate_record_info(MemoryContext mcxt, int ncolumns);
static bool JsObjectGetField(JsObject *obj, char *field, JsValue *jsv);
static void populate_recordset_record(PopulateRecordsetState *state, JsObject *obj);
-static void populate_array_json(PopulateArrayContext *ctx, char *json, int len);
-static void populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
+static bool populate_array_json(PopulateArrayContext *ctx, char *json, int len);
+static bool populate_array_dim_jsonb(PopulateArrayContext *ctx, JsonbValue *jbv,
int ndim);
static void populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim);
-static void populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
-static void populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
-static void populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
+static bool populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims);
+static bool populate_array_check_dimension(PopulateArrayContext *ctx, int ndim);
+static bool populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv);
static Datum populate_array(ArrayIOData *aio, const char *colname,
- MemoryContext mcxt, JsValue *jsv);
+ MemoryContext mcxt, JsValue *jsv,
+ bool *isnull,
+ Node *escontext);
static Datum populate_domain(DomainIOData *io, Oid typid, const char *colname,
- MemoryContext mcxt, JsValue *jsv, bool isnull);
+ MemoryContext mcxt, JsValue *jsv, bool isnull,
+ Node *escontext);
/* functions supporting jsonb_delete, jsonb_set and jsonb_concat */
static JsonbValue *IteratorConcat(JsonbIterator **it1, JsonbIterator **it2,
@@ -2484,12 +2491,12 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
if (ndim <= 0)
{
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the value of key \"%s\".", ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array")));
}
@@ -2506,13 +2513,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
appendStringInfo(&indices, "[%d]", ctx->sizes[i]);
if (ctx->colname)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s of key \"%s\".",
indices.data, ctx->colname)));
else
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("expected JSON array"),
errhint("See the array element %s.",
@@ -2520,8 +2527,13 @@ populate_array_report_expected_array(PopulateArrayContext *ctx, int ndim)
}
}
-/* set the number of dimensions of the populated array when it becomes known */
-static void
+/*
+ * Validate and set ndims for populating an array with some
+ * populate_array_*() function.
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
{
int i;
@@ -2529,7 +2541,12 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
Assert(ctx->ndims <= 0);
if (ndims <= 0)
+ {
populate_array_report_expected_array(ctx, ndims);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
ctx->ndims = ndims;
ctx->dims = palloc(sizeof(int) * ndims);
@@ -2537,10 +2554,16 @@ populate_array_assign_ndims(PopulateArrayContext *ctx, int ndims)
for (i = 0; i < ndims; i++)
ctx->dims[i] = -1; /* dimensions are unknown yet */
+
+ return true;
}
-/* check the populated subarray dimension */
-static void
+/*
+ * Check the populated subarray dimension
+ *
+ * Returns false if the input (ndims) is erratic.
+ */
+static bool
populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
{
int dim = ctx->sizes[ndim]; /* current dimension counter */
@@ -2548,21 +2571,31 @@ populate_array_check_dimension(PopulateArrayContext *ctx, int ndim)
if (ctx->dims[ndim] == -1)
ctx->dims[ndim] = dim; /* assign dimension if not yet known */
else if (ctx->dims[ndim] != dim)
- ereport(ERROR,
+ errsave(ctx->escontext,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("malformed JSON array"),
errdetail("Multidimensional arrays must have "
"sub-arrays with matching dimensions.")));
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
+
/* reset the current array dimension size counter */
ctx->sizes[ndim] = 0;
/* increment the parent dimension counter if it is a nested sub-array */
if (ndim > 0)
ctx->sizes[ndim - 1]++;
+
+ return true;
}
-static void
+/*
+ * Returns true if the array element value was successfully extracted from jsv
+ * and added to ctx->astate. False if an error occurred when doing so.
+ */
+static bool
populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
{
Datum element;
@@ -2573,13 +2606,18 @@ populate_array_element(PopulateArrayContext *ctx, int ndim, JsValue *jsv)
ctx->aio->element_type,
ctx->aio->element_typmod,
NULL, ctx->mcxt, PointerGetDatum(NULL),
- jsv, &element_isnull);
+ jsv, &element_isnull, ctx->escontext);
+ /* Nothing to do on an error. */
+ if (SOFT_ERROR_OCCURRED(ctx->escontext))
+ return false;
accumArrayResult(ctx->astate, element, element_isnull,
ctx->aio->element_type, ctx->acxt);
Assert(ndim > 0);
ctx->sizes[ndim - 1]++; /* increment current dimension counter */
+
+ return true;
}
/* json object start handler for populate_array_json() */
@@ -2592,7 +2630,12 @@ populate_array_object_start(void *_state)
if (state->ctx->ndims <= 0)
populate_array_assign_ndims(state->ctx, ndim);
else if (ndim < state->ctx->ndims)
+ {
populate_array_report_expected_array(state->ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2609,7 +2652,11 @@ populate_array_array_end(void *_state)
populate_array_assign_ndims(ctx, ndim + 1);
if (ndim < ctx->ndims)
- populate_array_check_dimension(ctx, ndim);
+ {
+ /* Report if an error occurred. */
+ if (!populate_array_check_dimension(ctx, ndim))
+ return JSON_SEM_ACTION_FAILED;
+ }
return JSON_SUCCESS;
}
@@ -2667,7 +2714,9 @@ populate_array_element_end(void *_state, bool isnull)
state->element_start) * sizeof(char);
}
- populate_array_element(ctx, ndim, &jsv);
+ /* Report if an error occurred. */
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return JSON_SEM_ACTION_FAILED;
}
return JSON_SUCCESS;
@@ -2684,7 +2733,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
if (ctx->ndims <= 0)
populate_array_assign_ndims(ctx, ndim);
else if (ndim < ctx->ndims)
+ {
populate_array_report_expected_array(ctx, ndim);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(state->ctx->escontext));
+ return JSON_SEM_ACTION_FAILED;
+ }
if (ndim == ctx->ndims)
{
@@ -2697,8 +2751,12 @@ populate_array_scalar(void *_state, char *token, JsonTokenType tokentype)
return JSON_SUCCESS;
}
-/* parse a json array and populate array */
-static void
+/*
+ * Parse a json array and populate array
+ *
+ * Returns false if an error occurs when parsing.
+ */
+static bool
populate_array_json(PopulateArrayContext *ctx, char *json, int len)
{
PopulateArrayState state;
@@ -2716,19 +2774,25 @@ populate_array_json(PopulateArrayContext *ctx, char *json, int len)
sem.array_element_end = populate_array_element_end;
sem.scalar = populate_array_scalar;
- pg_parse_json_or_ereport(state.lex, &sem);
-
- /* number of dimensions should be already known */
- Assert(ctx->ndims > 0 && ctx->dims);
+ if (pg_parse_json_or_errsave(state.lex, &sem, ctx->escontext))
+ {
+ /* number of dimensions should be already known */
+ Assert(ctx->ndims > 0 && ctx->dims);
+ }
freeJsonLexContext(state.lex);
+
+ return !SOFT_ERROR_OCCURRED(ctx->escontext);
}
/*
* populate_array_dim_jsonb() -- Iterate recursively through jsonb sub-array
* elements and accumulate result using given ArrayBuildState.
+ *
+ * Returns false if we return partway through because of an error in a
+ * subroutine.
*/
-static void
+static bool
populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
JsonbValue *jbv, /* jsonb sub-array */
int ndim) /* current dimension */
@@ -2742,7 +2806,12 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ {
populate_array_report_expected_array(ctx, ndim - 1);
+ /* Getting here means the error was reported softly. */
+ Assert(SOFT_ERROR_OCCURRED(ctx->escontext));
+ return false;
+ }
Assert(!JsonContainerIsScalar(jbc));
@@ -2763,7 +2832,10 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
(tok == WJB_ELEM &&
(val.type != jbvBinary ||
!JsonContainerIsArray(val.val.binary.data)))))
- populate_array_assign_ndims(ctx, ndim);
+ {
+ if (!populate_array_assign_ndims(ctx, ndim))
+ return false;
+ }
jsv.is_json = false;
jsv.val.jsonb = &val;
@@ -2776,16 +2848,21 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
* it is not the innermost dimension.
*/
if (ctx->ndims > 0 && ndim >= ctx->ndims)
- populate_array_element(ctx, ndim, &jsv);
+ {
+ if (!populate_array_element(ctx, ndim, &jsv))
+ return false;
+ }
else
{
/* populate child sub-array */
- populate_array_dim_jsonb(ctx, &val, ndim + 1);
+ if (!populate_array_dim_jsonb(ctx, &val, ndim + 1))
+ return false;
/* number of dimensions should be already known */
Assert(ctx->ndims > 0 && ctx->dims);
- populate_array_check_dimension(ctx, ndim);
+ if (!populate_array_check_dimension(ctx, ndim))
+ return false;
}
tok = JsonbIteratorNext(&it, &val, true);
@@ -2796,14 +2873,22 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
/* free iterator, iterating until WJB_DONE */
tok = JsonbIteratorNext(&it, &val, true);
Assert(tok == WJB_DONE && !it);
+
+ return true;
}
-/* recursively populate an array from json/jsonb */
+/*
+ * Recursively populate an array from json/jsonb
+ *
+ * *isnull is set to true if an error is reported during parsing.
+ */
static Datum
populate_array(ArrayIOData *aio,
const char *colname,
MemoryContext mcxt,
- JsValue *jsv)
+ JsValue *jsv,
+ bool *isnull,
+ Node *escontext)
{
PopulateArrayContext ctx;
Datum result;
@@ -2818,14 +2903,27 @@ populate_array(ArrayIOData *aio,
ctx.ndims = 0; /* unknown yet */
ctx.dims = NULL;
ctx.sizes = NULL;
+ ctx.escontext = escontext;
if (jsv->is_json)
- populate_array_json(&ctx, jsv->val.json.str,
- jsv->val.json.len >= 0 ? jsv->val.json.len
- : strlen(jsv->val.json.str));
+ {
+ /* Return null if an error was found. */
+ if (!populate_array_json(&ctx, jsv->val.json.str,
+ jsv->val.json.len >= 0 ? jsv->val.json.len
+ : strlen(jsv->val.json.str)))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
+ }
else
{
- populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1);
+ /* Return null if an error was found. */
+ if (!populate_array_dim_jsonb(&ctx, jsv->val.jsonb, 1))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
ctx.dims[0] = ctx.sizes[0];
}
@@ -2843,11 +2941,16 @@ populate_array(ArrayIOData *aio,
pfree(ctx.sizes);
pfree(lbs);
+ *isnull = false;
return result;
}
-static void
-JsValueToJsObject(JsValue *jsv, JsObject *jso)
+/*
+ * Returns false if an error occurs, provided escontext points to an
+ * ErrorSaveContext.
+ */
+static bool
+JsValueToJsObject(JsValue *jsv, JsObject *jso, Node *escontext)
{
jso->is_json = jsv->is_json;
@@ -2859,7 +2962,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
jsv->val.json.len >= 0
? jsv->val.json.len
: strlen(jsv->val.json.str),
- "populate_composite");
+ "populate_composite",
+ escontext);
}
else
{
@@ -2877,7 +2981,7 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
is_scalar = IsAJsonbScalar(jbv) ||
(jbv->type == jbvBinary &&
JsonContainerIsScalar(jbv->val.binary.data));
- ereport(ERROR,
+ errsave(escontext,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
is_scalar
? errmsg("cannot call %s on a scalar",
@@ -2886,6 +2990,8 @@ JsValueToJsObject(JsValue *jsv, JsObject *jso)
"populate_composite")));
}
}
+
+ return !SOFT_ERROR_OCCURRED(escontext);
}
/* acquire or update cached tuple descriptor for a composite type */
@@ -2912,7 +3018,12 @@ update_cached_tupdesc(CompositeIOData *io, MemoryContext mcxt)
}
}
-/* recursively populate a composite (row type) value from json/jsonb */
+/*
+ * Recursively populate a composite (row type) value from json/jsonb
+ *
+ * Returns null if an error occurs in a subroutine, provided escontext points
+ * to an ErrorSaveContext.
+ */
static Datum
populate_composite(CompositeIOData *io,
Oid typid,
@@ -2920,14 +3031,15 @@ populate_composite(CompositeIOData *io,
MemoryContext mcxt,
HeapTupleHeader defaultval,
JsValue *jsv,
- bool isnull)
+ bool *isnull,
+ Node *escontext)
{
Datum result;
/* acquire/update cached tuple descriptor */
update_cached_tupdesc(io, mcxt);
- if (isnull)
+ if (*isnull)
result = (Datum) 0;
else
{
@@ -2935,11 +3047,15 @@ populate_composite(CompositeIOData *io,
JsObject jso;
/* prepare input value */
- JsValueToJsObject(jsv, &jso);
+ if (!JsValueToJsObject(jsv, &jso, escontext))
+ {
+ *isnull = true;
+ return (Datum) 0;
+ }
/* populate resulting record tuple */
tuple = populate_record(io->tupdesc, &io->record_io,
- defaultval, mcxt, &jso);
+ defaultval, mcxt, &jso, escontext);
result = HeapTupleHeaderGetDatum(tuple);
JsObjectFree(&jso);
@@ -2951,14 +3067,20 @@ populate_composite(CompositeIOData *io,
* now, we can tell by comparing typid to base_typid.)
*/
if (typid != io->base_typid && typid != RECORDOID)
- domain_check(result, isnull, typid, &io->domain_info, mcxt);
+ domain_check(result, *isnull, typid, &io->domain_info, mcxt);
return result;
}
-/* populate non-null scalar value from json/jsonb value */
+/*
+ * Populate non-null scalar value from json/jsonb value.
+ *
+ * Returns null if an error occurs during the call to type input function,
+ * provided escontext is valid.
+ */
static Datum
-populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
+populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv,
+ bool *isnull, Node *escontext)
{
Datum res;
char *str = NULL;
@@ -3029,7 +3151,12 @@ populate_scalar(ScalarIOData *io, Oid typid, int32 typmod, JsValue *jsv)
elog(ERROR, "unrecognized jsonb type: %d", (int) jbv->type);
}
- res = InputFunctionCall(&io->typiofunc, str, io->typioparam, typmod);
+ if (!InputFunctionCallSafe(&io->typiofunc, str, io->typioparam, typmod,
+ escontext, &res))
+ {
+ res = (Datum) 0;
+ *isnull = true;
+ }
/* free temporary buffer */
if (str != json)
@@ -3044,7 +3171,8 @@ populate_domain(DomainIOData *io,
const char *colname,
MemoryContext mcxt,
JsValue *jsv,
- bool isnull)
+ bool isnull,
+ Node *escontext)
{
Datum res;
@@ -3055,8 +3183,8 @@ populate_domain(DomainIOData *io,
res = populate_record_field(io->base_io,
io->base_typid, io->base_typmod,
colname, mcxt, PointerGetDatum(NULL),
- jsv, &isnull);
- Assert(!isnull);
+ jsv, &isnull, escontext);
+ Assert(!isnull || SOFT_ERROR_OCCURRED(escontext));
}
domain_check(res, isnull, typid, &io->domain_info, mcxt);
@@ -3160,7 +3288,8 @@ populate_record_field(ColumnIOData *col,
MemoryContext mcxt,
Datum defaultval,
JsValue *jsv,
- bool *isnull)
+ bool *isnull,
+ Node *escontext)
{
TypeCat typcat;
@@ -3193,10 +3322,12 @@ populate_record_field(ColumnIOData *col,
switch (typcat)
{
case TYPECAT_SCALAR:
- return populate_scalar(&col->scalar_io, typid, typmod, jsv);
+ return populate_scalar(&col->scalar_io, typid, typmod, jsv,
+ isnull, escontext);
case TYPECAT_ARRAY:
- return populate_array(&col->io.array, colname, mcxt, jsv);
+ return populate_array(&col->io.array, colname, mcxt, jsv,
+ isnull, escontext);
case TYPECAT_COMPOSITE:
case TYPECAT_COMPOSITE_DOMAIN:
@@ -3205,11 +3336,12 @@ populate_record_field(ColumnIOData *col,
DatumGetPointer(defaultval)
? DatumGetHeapTupleHeader(defaultval)
: NULL,
- jsv, *isnull);
+ jsv, isnull,
+ escontext);
case TYPECAT_DOMAIN:
return populate_domain(&col->io.domain, typid, colname, mcxt,
- jsv, *isnull);
+ jsv, *isnull, escontext);
default:
elog(ERROR, "unrecognized type category '%c'", typcat);
@@ -3266,7 +3398,8 @@ populate_record(TupleDesc tupdesc,
RecordIOData **record_p,
HeapTupleHeader defaultval,
MemoryContext mcxt,
- JsObject *obj)
+ JsObject *obj,
+ Node *escontext)
{
RecordIOData *record = *record_p;
Datum *values;
@@ -3358,7 +3491,8 @@ populate_record(TupleDesc tupdesc,
mcxt,
nulls[i] ? (Datum) 0 : values[i],
&field,
- &nulls[i]);
+ &nulls[i],
+ escontext);
}
res = heap_form_tuple(tupdesc, values, nulls);
@@ -3445,6 +3579,7 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
JsValue jsv = {0};
HeapTupleHeader rec;
Datum rettuple;
+ bool isnull;
JsonbValue jbv;
MemoryContext fnmcxt = fcinfo->flinfo->fn_mcxt;
PopulateRecordCache *cache = fcinfo->flinfo->fn_extra;
@@ -3531,8 +3666,11 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
jbv.val.binary.len = VARSIZE(jb) - VARHDRSZ;
}
+ isnull = false;
rettuple = populate_composite(&cache->c.io.composite, cache->argtype,
- NULL, fnmcxt, rec, &jsv, false);
+ NULL, fnmcxt, rec, &jsv, &isnull,
+ NULL);
+ Assert(!isnull);
PG_RETURN_DATUM(rettuple);
}
@@ -3543,7 +3681,8 @@ populate_record_worker(FunctionCallInfo fcinfo, const char *funcname,
* decompose a json object into a hash table.
*/
static HTAB *
-get_json_object_as_hash(char *json, int len, const char *funcname)
+get_json_object_as_hash(char *json, int len, const char *funcname,
+ Node *escontext)
{
HASHCTL ctl;
HTAB *tab;
@@ -3572,7 +3711,7 @@ get_json_object_as_hash(char *json, int len, const char *funcname)
sem->object_field_start = hash_object_field_start;
sem->object_field_end = hash_object_field_end;
- pg_parse_json_or_ereport(state->lex, sem);
+ pg_parse_json_or_errsave(state->lex, sem, escontext);
freeJsonLexContext(state->lex);
@@ -3743,7 +3882,8 @@ populate_recordset_record(PopulateRecordsetState *state, JsObject *obj)
&cache->c.io.composite.record_io,
state->rec,
cache->fn_mcxt,
- obj);
+ obj,
+ NULL);
/* if it's domain over composite, check domain constraints */
if (cache->c.typcat == TYPECAT_COMPOSITE_DOMAIN)
--
2.35.3
[application/octet-stream] v25-0004-JSON_TABLE.patch (165.3K, ../../CA+HiwqF14cAgG7e0SJ90VXd8OKPzeRLBAn3p=-R8JV7H+M35fA@mail.gmail.com/5-v25-0004-JSON_TABLE.patch)
download | inline diff:
From 5331239b3ded79c1650f6eeaa8237ab6a9500015 Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:14 +0900
Subject: [PATCH v25 4/5] JSON_TABLE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This feature allows jsonb data to be treated as a table and thus
used in a FROM clause like other tabular data. Data can be selected
from the jsonb using jsonpath expressions, and hoisted out of nested
structures in the jsonb to form multiple rows, more or less like an
outer join.
This also adds the PLAN clauses for JSON_TABLE, which allow the user
to specify how data from nested paths are joined, allowing
considerable freedom in shaping the tabular output of JSON_TABLE.
PLAN DEFAULT allows the user to specify the global strategies when
dealing with sibling or child nested paths. The is often sufficient
to achieve the necessary goal, and is considerably simpler than the
full PLAN clause, which allows the user to specify the strategy to be
used for each named nested path.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 496 ++++++++
src/backend/commands/explain.c | 8 +-
src/backend/executor/execExprInterp.c | 6 +
src/backend/executor/nodeTableFuncscan.c | 27 +-
src/backend/nodes/makefuncs.c | 19 +
src/backend/nodes/nodeFuncs.c | 28 +
src/backend/parser/Makefile | 1 +
src/backend/parser/gram.y | 312 ++++-
src/backend/parser/meson.build | 1 +
src/backend/parser/parse_clause.c | 14 +-
src/backend/parser/parse_expr.c | 13 +
src/backend/parser/parse_jsontable.c | 760 ++++++++++++
src/backend/parser/parse_relation.c | 5 +-
src/backend/parser/parse_target.c | 3 +
src/backend/utils/adt/jsonpath_exec.c | 543 +++++++++
src/backend/utils/adt/ruleutils.c | 279 ++++-
src/include/nodes/execnodes.h | 2 +
src/include/nodes/makefuncs.h | 2 +
src/include/nodes/parsenodes.h | 89 ++
src/include/nodes/primnodes.h | 61 +-
src/include/parser/kwlist.h | 4 +
src/include/parser/parse_clause.h | 3 +
src/include/utils/jsonpath.h | 3 +
src/test/regress/expected/json_sqljson.out | 6 +
src/test/regress/expected/jsonb_sqljson.out | 1186 +++++++++++++++++++
src/test/regress/sql/json_sqljson.sql | 4 +
src/test/regress/sql/jsonb_sqljson.sql | 674 +++++++++++
src/tools/pgindent/typedefs.list | 13 +
28 files changed, 4535 insertions(+), 27 deletions(-)
create mode 100644 src/backend/parser/parse_jsontable.c
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index c12ee06713..39584fba58 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17214,6 +17214,502 @@ array w/o UK? | t
</table>
</sect2>
+ <sect2 id="functions-sqljson-table">
+ <title>JSON_TABLE</title>
+ <indexterm>
+ <primary>json_table</primary>
+ </indexterm>
+
+ <para>
+ <function>json_table</function> is an SQL/JSON function which
+ queries <acronym>JSON</acronym> data
+ and presents the results as a relational view, which can be accessed as a
+ regular SQL table. You can only use <function>json_table</function> inside the
+ <literal>FROM</literal> clause of a <literal>SELECT</literal> statement.
+ </para>
+
+ <para>
+ Taking JSON data as input, <function>json_table</function> uses
+ a path expression to extract a part of the provided data that
+ will be used as a <firstterm>row pattern</firstterm> for the
+ constructed view. Each SQL/JSON item at the top level of the row pattern serves
+ as the source for a separate row in the constructed relational view.
+ </para>
+
+ <para>
+ To split the row pattern into columns, <function>json_table</function>
+ provides the <literal>COLUMNS</literal> clause that defines the
+ schema of the created view. For each column to be constructed,
+ this clause provides a separate path expression that evaluates
+ the row pattern, extracts a JSON item, and returns it as a
+ separate SQL value for the specified column. If the required value
+ is stored in a nested level of the row pattern, it can be extracted
+ using the <literal>NESTED PATH</literal> subclause. Joining the
+ columns returned by <literal>NESTED PATH</literal> can add multiple
+ new rows to the constructed view. Such rows are called
+ <firstterm>child rows</firstterm>, as opposed to the <firstterm>parent row</firstterm>
+ that generates them.
+ </para>
+
+ <para>
+ The rows produced by <function>JSON_TABLE</function> are laterally
+ joined to the row that generated them, so you do not have to explicitly join
+ the constructed view with the original table holding <acronym>JSON</acronym>
+ data. Optionally, you can specify how to join the columns returned
+ by <literal>NESTED PATH</literal> using the <literal>PLAN</literal> clause.
+ </para>
+
+ <para>
+ Each <literal>NESTED PATH</literal> clause can generate one or more
+ columns. Columns produced by <literal>NESTED PATH</literal>s at the
+ same level are considered to be <firstterm>siblings</firstterm>,
+ while a column produced by a <literal>NESTED PATH</literal> is
+ considered to be a child of the column produced by a
+ <literal>NESTED PATH</literal> or row expression at a higher level.
+ Sibling columns are always joined first. Once they are processed,
+ the resulting rows are joined to the parent row.
+ </para>
+
+ <para>
+ The syntax is:
+ </para>
+
+<synopsis>
+JSON_TABLE (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> AS <replaceable>json_path_name</replaceable> </optional> <optional> PASSING { <replaceable>value</replaceable> AS <replaceable>varname</replaceable> } <optional>, ...</optional> </optional>
+ COLUMNS ( <replaceable class="parameter">json_table_column</replaceable> <optional>, ...</optional> )
+ <optional> { <literal>ERROR</literal> | <literal>EMPTY</literal> } <literal>ON ERROR</literal> </optional>
+ <optional>
+ PLAN ( <replaceable class="parameter">json_table_plan</replaceable> ) |
+ PLAN DEFAULT ( { INNER | OUTER } <optional> , { CROSS | UNION } </optional>
+ | { CROSS | UNION } <optional> , { INNER | OUTER } </optional> )
+ </optional>
+)
+
+<phrase>
+where <replaceable class="parameter">json_table_column</replaceable> is:
+</phrase>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> FORMAT <replaceable>json_representation</replaceable>
+ <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { WITHOUT | WITH { CONDITIONAL | <optional>UNCONDITIONAL</optional> } } <optional> ARRAY </optional> WRAPPER </optional>
+ <optional> { KEEP | OMIT } QUOTES <optional> ON SCALAR STRING </optional> </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON EMPTY </optional>
+ <optional> { ERROR | NULL | EMPTY { ARRAY | OBJECT } | DEFAULT <replaceable>expression</replaceable> } ON ERROR </optional>
+ | <replaceable>name</replaceable> <replaceable>type</replaceable> EXISTS <optional> PATH <replaceable>json_path_specification</replaceable> </optional>
+ <optional> { ERROR | TRUE | FALSE | UNKNOWN } ON ERROR </optional>
+ | NESTED PATH <replaceable>json_path_specification</replaceable> <optional> AS <replaceable>path_name</replaceable> </optional>
+ COLUMNS ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ | <replaceable>name</replaceable> FOR ORDINALITY
+<phrase>
+<replaceable>json_table_plan</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> <optional> { OUTER | INNER } <replaceable>json_table_plan_primary</replaceable> </optional>
+ | <replaceable>json_table_plan_primary</replaceable> { UNION <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+ | <replaceable>json_table_plan_primary</replaceable> { CROSS <replaceable>json_table_plan_primary</replaceable> } <optional>...</optional>
+<phrase>
+<replaceable>json_table_plan_primary</replaceable> is:
+</phrase>
+ <replaceable>json_path_name</replaceable> | ( <replaceable>json_table_plan</replaceable> )
+</synopsis>
+
+ <para>
+ Each syntax element is described below in more detail.
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional></literal>
+ </term>
+ <listitem>
+ <para>
+ The input data to query, the JSON path expression defining the query,
+ and an optional <literal>PASSING</literal> clause, which can provide data
+ values to the <replaceable>path_expression</replaceable>.
+ The result of the input data
+ evaluation is called the <firstterm>row pattern</firstterm>. The row
+ pattern is used as the source for row values in the constructed view.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>COLUMNS</literal>( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ The <literal>COLUMNS</literal> clause defining the schema of the
+ constructed view. In this clause, you must specify all the columns
+ to be filled with SQL/JSON items.
+ The <replaceable>json_table_column</replaceable>
+ expression has the following syntax variants:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal><replaceable>name</replaceable> <replaceable>type</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional></literal>
+ </term>
+ <listitem>
+
+ <para>
+ Inserts a single SQL/JSON item into each row of
+ the specified column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON EMPTY</literal> and
+ <literal>ON ERROR</literal> clauses to define how to handle missing values
+ or structural errors.
+ <literal>WRAPPER</literal> and <literal>QUOTES</literal> clauses can only
+ be used with JSON, array, and composite types.
+ These clauses have the same syntax and semantics as for
+ <function>json_value</function> and <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable> <literal>FORMAT</literal> <replaceable>json_representation</replaceable>
+ <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a composite SQL/JSON
+ item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated and
+ the column is filled with the produced SQL/JSON items, one for each row.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ In this case, the column name must correspond to one of the
+ keys within the SQL/JSON item produced by the row pattern.
+ </para>
+ <para>
+ Optionally, you can add <literal>WRAPPER</literal>, <literal>QUOTES</literal>,
+ <literal>ON EMPTY</literal> and <literal>ON ERROR</literal> clauses
+ to define additional settings for the returned SQL/JSON items.
+ These clauses have the same syntax and semantics as
+ for <function>json_query</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <replaceable>type</replaceable>
+ <literal>EXISTS</literal> <optional> <literal>PATH</literal> <replaceable>json_path_specification</replaceable> </optional>
+ </term>
+ <listitem>
+
+ <para>
+ Generates a column and inserts a boolean item into each row of this column.
+ </para>
+ <para>
+ The provided <literal>PATH</literal> expression is evaluated,
+ a check whether any SQL/JSON items were returned is done, and
+ the column is filled with the resulting boolean value, one for each row.
+ The specified <parameter>type</parameter> should have a cast from the
+ <type>boolean</type>.
+ If the <literal>PATH</literal> expression is omitted,
+ <function>JSON_TABLE</function> uses the
+ <literal>$.<replaceable>name</replaceable></literal> path expression,
+ where <replaceable>name</replaceable> is the provided column name.
+ </para>
+ <para>
+ Optionally, you can add <literal>ON ERROR</literal> clause to define
+ error behavior. This clause has the same syntax and semantics as
+ for <function>json_exists</function>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>NESTED PATH</literal> <replaceable>json_path_specification</replaceable> <optional> <literal>AS</literal> <replaceable>json_path_name</replaceable> </optional>
+ <literal>COLUMNS</literal> ( <replaceable>json_table_column</replaceable> <optional>, ...</optional> )
+ </term>
+ <listitem>
+
+ <para>
+ Extracts SQL/JSON items from nested levels of the row pattern,
+ generates one or more columns as defined by the <literal>COLUMNS</literal>
+ subclause, and inserts the extracted SQL/JSON items into each row of these columns.
+ The <replaceable>json_table_column</replaceable> expression in the
+ <literal>COLUMNS</literal> subclause uses the same syntax as in the
+ parent <literal>COLUMNS</literal> clause.
+ </para>
+
+ <para>
+ The <literal>NESTED PATH</literal> syntax is recursive,
+ so you can go down multiple nested levels by specifying several
+ <literal>NESTED PATH</literal> subclauses within each other.
+ It allows to unnest the hierarchy of JSON objects and arrays
+ in a single function invocation rather than chaining several
+ <function>JSON_TABLE</function> expressions in an SQL statement.
+ </para>
+
+ <para>
+ You can use the <literal>PLAN</literal> clause to define how
+ to join the columns returned by <literal>NESTED PATH</literal> clauses.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <replaceable>name</replaceable> <literal>FOR ORDINALITY</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Adds an ordinality column that provides sequential row numbering.
+ You can have only one ordinality column per table. Row numbering
+ is 1-based. For child rows that result from the <literal>NESTED PATH</literal>
+ clauses, the parent row number is repeated.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>AS</literal> <replaceable>json_path_name</replaceable>
+ </term>
+ <listitem>
+
+ <para>
+ The optional <replaceable>json_path_name</replaceable> serves as an
+ identifier of the provided <replaceable>json_path_specification</replaceable>.
+ The path name must be unique and distinct from the column names.
+ When using the <literal>PLAN</literal> clause, you must specify the names
+ for all the paths, including the row pattern. Each path name can appear in
+ the <literal>PLAN</literal> clause only once.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN</literal> ( <replaceable>json_table_plan</replaceable> )
+ </term>
+ <listitem>
+
+ <para>
+ Defines how to join the data returned by <literal>NESTED PATH</literal>
+ clauses to the constructed view.
+ </para>
+ <para>
+ To join columns with parent/child relationship, you can use:
+ </para>
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>INNER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>INNER JOIN</literal>, so that the parent row
+ is omitted from the output if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>OUTER</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Use <literal>LEFT OUTER JOIN</literal>, so that the parent row
+ is always included into the output even if it does not have any child rows
+ after joining the data returned by <literal>NESTED PATH</literal>, with NULL values
+ inserted into the child columns if the corresponding
+ values are missing.
+ </para>
+ <para>
+ This is the default option for joining columns with parent/child relationship.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>
+ To join sibling columns, you can use:
+ </para>
+
+ <variablelist>
+ <varlistentry>
+ <term>
+ <literal>UNION</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each value produced by each of the sibling
+ columns. The columns from the other siblings are set to null.
+ </para>
+ <para>
+ This is the default option for joining sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>CROSS</literal>
+ </term>
+ <listitem>
+
+ <para>
+ Generate one row for each combination of values from the sibling columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ </variablelist>
+
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
+ <literal>PLAN DEFAULT</literal> ( <literal><replaceable>OUTER | INNER</replaceable> <optional>, <replaceable>UNION | CROSS</replaceable> </optional></literal> )
+ </term>
+ <listitem>
+ <para>
+ The terms can also be specified in reverse order. The
+ <literal>INNER</literal> or <literal>OUTER</literal> option defines the
+ joining plan for parent/child columns, while <literal>UNION</literal> or
+ <literal>CROSS</literal> affects joins of sibling columns. This form
+ of <literal>PLAN</literal> overrides the default plan for
+ all columns at once. Even though the path names are not included in the
+ <literal>PLAN DEFAULT</literal> form, to conform to the SQL/JSON standard
+ they must be provided for all the paths if the <literal>PLAN</literal>
+ clause is used.
+ </para>
+ <para>
+ <literal>PLAN DEFAULT</literal> is simpler than specifying a complete
+ <literal>PLAN</literal>, and is often all that is required to get the desired
+ output.
+ </para>
+ </listitem>
+ </varlistentry>
+ </variablelist>
+
+ <para>Examples</para>
+
+ <para>
+ In these examples the following small table storing some JSON data will be used:
+<programlisting>
+CREATE TABLE my_films ( js jsonb );
+
+INSERT INTO my_films VALUES (
+'{ "favorites" : [
+ { "kind" : "comedy", "films" : [
+ { "title" : "Bananas",
+ "director" : "Woody Allen"},
+ { "title" : "The Dinner Game",
+ "director" : "Francis Veber" } ] },
+ { "kind" : "horror", "films" : [
+ { "title" : "Psycho",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "thriller", "films" : [
+ { "title" : "Vertigo",
+ "director" : "Alfred Hitchcock" } ] },
+ { "kind" : "drama", "films" : [
+ { "title" : "Yojimbo",
+ "director" : "Akira Kurosawa" } ] }
+ ] }');
+</programlisting>
+ </para>
+ <para>
+ Query the <structname>my_films</structname> table holding
+ some JSON data about the films and create a view that
+ distributes the film genre, title, and director between separate columns:
+<screen>
+SELECT jt.* FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites[*]' COLUMNS (
+ id FOR ORDINALITY,
+ kind text PATH '$.kind',
+ NESTED PATH '$.films[*]' COLUMNS (
+ title text PATH '$.title',
+ director text PATH '$.director'))) AS jt;
+----+----------+------------------+-------------------
+ id | kind | title | director
+----+----------+------------------+-------------------
+ 1 | comedy | Bananas | Woody Allen
+ 1 | comedy | The Dinner Game | Francis Veber
+ 2 | horror | Psycho | Alfred Hitchcock
+ 3 | thriller | Vertigo | Alfred Hitchcock
+ 4 | drama | Yojimbo | Akira Kurosawa
+ (5 rows)
+</screen>
+ </para>
+
+ <para>
+ Find a director that has done films in two different genres:
+<screen>
+SELECT
+ director1 AS director, title1, kind1, title2, kind2
+FROM
+ my_films,
+ JSON_TABLE ( js, '$.favorites' AS favs COLUMNS (
+ NESTED PATH '$[*]' AS films1 COLUMNS (
+ kind1 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film1 COLUMNS (
+ title1 text PATH '$.title',
+ director1 text PATH '$.director')
+ ),
+ NESTED PATH '$[*]' AS films2 COLUMNS (
+ kind2 text PATH '$.kind',
+ NESTED PATH '$.films[*]' AS film2 COLUMNS (
+ title2 text PATH '$.title',
+ director2 text PATH '$.director'
+ )
+ )
+ )
+ PLAN (favs OUTER ((films1 INNER film1) CROSS (films2 INNER film2)))
+ ) AS jt
+ WHERE kind1 > kind2 AND director1 = director2;
+
+ director | title1 | kind1 | title2 | kind2
+------------------+---------+----------+--------+--------
+ Alfred Hitchcock | Vertigo | thriller | Psycho | horror
+(1 row)
+</screen>
+ </para>
+ </sect2>
+
<sect2 id="functions-sqljson-path">
<title>The SQL/JSON Path Language</title>
diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index f1d71bc54e..8e35525781 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -3892,7 +3892,13 @@ ExplainTargetRel(Plan *plan, Index rti, ExplainState *es)
break;
case T_TableFuncScan:
Assert(rte->rtekind == RTE_TABLEFUNC);
- objectname = "xmltable";
+ if (rte->tablefunc)
+ if (rte->tablefunc->functype == TFT_XMLTABLE)
+ objectname = "xmltable";
+ else /* Must be TFT_JSON_TABLE */
+ objectname = "json_table";
+ else
+ objectname = NULL;
objecttag = "Table Function Name";
break;
case T_ValuesScan:
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 74217fb59d..59d68f8298 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -4364,6 +4364,12 @@ ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
break;
}
+ case JSON_TABLE_OP:
+ post_eval->jump_eval_coercion = -1;
+ *op->resvalue = item;
+ *op->resnull = false;
+ break;
+
default:
elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
return false;
diff --git a/src/backend/executor/nodeTableFuncscan.c b/src/backend/executor/nodeTableFuncscan.c
index a60dcd4943..0d7f518afd 100644
--- a/src/backend/executor/nodeTableFuncscan.c
+++ b/src/backend/executor/nodeTableFuncscan.c
@@ -28,6 +28,7 @@
#include "miscadmin.h"
#include "nodes/execnodes.h"
#include "utils/builtins.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/xml.h"
@@ -161,8 +162,9 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
scanstate->ss.ps.qual =
ExecInitQual(node->scan.plan.qual, &scanstate->ss.ps);
- /* Only XMLTABLE is supported currently */
- scanstate->routine = &XmlTableRoutine;
+ /* Only XMLTABLE and JSON_TABLE are supported currently */
+ scanstate->routine =
+ tf->functype == TFT_XMLTABLE ? &XmlTableRoutine : &JsonbTableRoutine;
scanstate->perTableCxt =
AllocSetContextCreate(CurrentMemoryContext,
@@ -182,6 +184,10 @@ ExecInitTableFuncScan(TableFuncScan *node, EState *estate, int eflags)
ExecInitExprList(tf->colexprs, (PlanState *) scanstate);
scanstate->coldefexprs =
ExecInitExprList(tf->coldefexprs, (PlanState *) scanstate);
+ scanstate->colvalexprs =
+ ExecInitExprList(tf->colvalexprs, (PlanState *) scanstate);
+ scanstate->passingvalexprs =
+ ExecInitExprList(tf->passingvalexprs, (PlanState *) scanstate);
scanstate->notnulls = tf->notnulls;
@@ -369,14 +375,17 @@ tfuncInitialize(TableFuncScanState *tstate, ExprContext *econtext, Datum doc)
routine->SetNamespace(tstate, ns_name, ns_uri);
}
- /* Install the row filter expression into the table builder context */
- value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
- if (isnull)
- ereport(ERROR,
- (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
- errmsg("row filter expression must not be null")));
+ if (routine->SetRowFilter)
+ {
+ /* Install the row filter expression into the table builder context */
+ value = ExecEvalExpr(tstate->rowexpr, econtext, &isnull);
+ if (isnull)
+ ereport(ERROR,
+ (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
+ errmsg("row filter expression must not be null")));
- routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ routine->SetRowFilter(tstate, TextDatumGetCString(value));
+ }
/*
* Install the column filter expressions into the table builder context.
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index 746ed3d1da..2ad7b8cb7b 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -890,6 +890,25 @@ makeJsonBehavior(JsonBehaviorType type, Node *default_expr, int location)
return behavior;
}
+/*
+ * makeJsonTableJoinedPlan -
+ * creates a joined JsonTablePlan node
+ */
+Node *
+makeJsonTableJoinedPlan(JsonTablePlanJoinType type, Node *plan1, Node *plan2,
+ int location)
+{
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_JOINED;
+ n->join_type = type;
+ n->plan1 = castNode(JsonTablePlan, plan1);
+ n->plan2 = castNode(JsonTablePlan, plan2);
+ n->location = location;
+
+ return (Node *) n;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 8e49ef6b95..df951c212c 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -2625,6 +2625,10 @@ expression_tree_walker_impl(Node *node,
return true;
if (WALK(tf->coldefexprs))
return true;
+ if (WALK(tf->colvalexprs))
+ return true;
+ if (WALK(tf->passingvalexprs))
+ return true;
}
break;
default:
@@ -3685,6 +3689,8 @@ expression_tree_mutator_impl(Node *node,
MUTATE(newnode->rowexpr, tf->rowexpr, Node *);
MUTATE(newnode->colexprs, tf->colexprs, List *);
MUTATE(newnode->coldefexprs, tf->coldefexprs, List *);
+ MUTATE(newnode->colvalexprs, tf->colvalexprs, List *);
+ MUTATE(newnode->passingvalexprs, tf->passingvalexprs, List *);
return (Node *) newnode;
}
break;
@@ -4124,6 +4130,28 @@ raw_expression_tree_walker_impl(Node *node,
return true;
}
break;
+ case T_JsonTable:
+ {
+ JsonTable *jt = (JsonTable *) node;
+
+ if (WALK(jt->common))
+ return true;
+ if (WALK(jt->columns))
+ return true;
+ }
+ break;
+ case T_JsonTableColumn:
+ {
+ JsonTableColumn *jtc = (JsonTableColumn *) node;
+
+ if (WALK(jtc->typeName))
+ return true;
+ if (WALK(jtc->behavior))
+ return true;
+ if (jtc->coltype == JTC_NESTED && WALK(jtc->columns))
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/parser/Makefile b/src/backend/parser/Makefile
index 401c16686c..3162a01f30 100644
--- a/src/backend/parser/Makefile
+++ b/src/backend/parser/Makefile
@@ -23,6 +23,7 @@ OBJS = \
parse_enr.o \
parse_expr.o \
parse_func.o \
+ parse_jsontable.o \
parse_merge.o \
parse_node.o \
parse_oper.o \
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 1dc3300fde..56895cf47d 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -655,13 +655,36 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_aggregate_func
json_api_common_syntax
json_argument
+ json_table
+ json_table_column_definition
+ json_table_ordinality_column_definition
+ json_table_regular_column_definition
+ json_table_formatted_column_definition
+ json_table_exists_column_definition
+ json_table_nested_columns
+ json_table_plan_clause_opt
+ json_table_plan
+ json_table_plan_simple
+ json_table_plan_parent_child
+ json_table_plan_outer
+ json_table_plan_inner
+ json_table_plan_sibling
+ json_table_plan_union
+ json_table_plan_cross
+ json_table_plan_primary
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
json_arguments
+ json_table_columns_clause
+ json_table_column_definition_list
+%type <str> json_table_column_path_specification_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
+ json_table_default_plan_choices
+ json_table_default_plan_inner_outer
+ json_table_default_plan_union_cross
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
@@ -733,7 +756,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
- JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_TABLE JSON_VALUE
KEY KEYS KEEP
@@ -744,8 +767,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
MINUTE_P MINVALUE MODE MONTH_P MOVE
- NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NFC NFD NFKC NFKD NO NONE
- NORMALIZE NORMALIZED
+ NAME_P NAMES NATIONAL NATURAL NCHAR NESTED NEW NEXT NFC NFD NFKC NFKD NO
+ NONE NORMALIZE NORMALIZED
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
@@ -753,8 +776,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
- PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD
- PLACING PLANS POLICY
+ PARALLEL PARAMETER PARSER PARTIAL PARTITION PASSING PASSWORD PATH
+ PLACING PLAN PLANS POLICY
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
@@ -862,6 +885,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
* Using the same precedence as IDENT seems right for the reasons given above.
*/
%nonassoc UNBOUNDED /* ideally would have same precedence as IDENT */
+%nonassoc COLUMNS
%nonassoc IDENT PARTITION RANGE ROWS GROUPS PRECEDING FOLLOWING CUBE ROLLUP
%left Op OPERATOR /* multi-character ops and user-defined operators */
%left '+' '-'
@@ -884,6 +908,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
*/
%left JOIN CROSS LEFT FULL RIGHT INNER_P NATURAL
+%nonassoc json_table_column
+%nonassoc NESTED
+%left PATH
%%
/*
@@ -13410,6 +13437,21 @@ table_ref: relation_expr opt_alias_clause
$2->alias = $4;
$$ = (Node *) $2;
}
+ | json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $1);
+
+ jt->alias = $2;
+ $$ = (Node *) jt;
+ }
+ | LATERAL_P json_table opt_alias_clause
+ {
+ JsonTable *jt = castNode(JsonTable, $2);
+
+ jt->alias = $3;
+ jt->lateral = true;
+ $$ = (Node *) jt;
+ }
;
@@ -13977,6 +14019,8 @@ xmltable_column_option_el:
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(true), @1); }
| NULL_P
{ $$ = makeDefElem("is_not_null", (Node *) makeBoolean(false), @1); }
+ | PATH b_expr
+ { $$ = makeDefElem("path", $2, @1); }
;
xml_namespace_list:
@@ -16657,6 +16701,256 @@ json_quotes_clause_opt:
| /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
;
+json_table:
+ JSON_TABLE '('
+ json_api_common_syntax
+ json_table_columns_clause
+ json_table_plan_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonTable *n = makeNode(JsonTable);
+
+ n->common = (JsonCommon *) $3;
+ n->columns = $4;
+ n->plan = (JsonTablePlan *) $5;
+ n->behavior = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_columns_clause:
+ COLUMNS '(' json_table_column_definition_list ')' { $$ = $3; }
+ ;
+
+json_table_column_definition_list:
+ json_table_column_definition
+ { $$ = list_make1($1); }
+ | json_table_column_definition_list ',' json_table_column_definition
+ { $$ = lappend($1, $3); }
+ ;
+
+json_table_column_definition:
+ json_table_ordinality_column_definition %prec json_table_column
+ | json_table_regular_column_definition %prec json_table_column
+ | json_table_formatted_column_definition %prec json_table_column
+ | json_table_exists_column_definition %prec json_table_column
+ | json_table_nested_columns
+ ;
+
+json_table_ordinality_column_definition:
+ ColId FOR ORDINALITY
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FOR_ORDINALITY;
+ n->name = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_regular_column_definition:
+ ColId Typename
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_REGULAR;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->pathspec = $3;
+ n->wrapper = $4; /* disallowed during parse-analysis! */
+ n->quotes = $5; /* disallowed during parse-analysis! */
+ n->behavior = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_formatted_column_definition:
+ ColId Typename FORMAT_LA JSON json_encoding_clause_opt
+ json_table_column_path_specification_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_FORMATTED;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_JSON, $5, @1);
+ n->pathspec = $6;
+ n->wrapper = $7;
+ if (n->wrapper != JSW_NONE && $8 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@7)));
+ n->quotes = $8;
+ n->behavior = $9;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_exists_column_definition:
+ ColId Typename
+ EXISTS json_table_column_path_specification_clause_opt
+ json_behavior_clause_opt
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_EXISTS;
+ n->name = $1;
+ n->typeName = $2;
+ n->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ n->wrapper = JSW_NONE;
+ n->quotes = JS_QUOTES_UNSPEC;
+ n->pathspec = $4;
+ n->behavior = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_column_path_specification_clause_opt:
+ PATH Sconst { $$ = $2; }
+ | /* EMPTY */ %prec json_table_column { $$ = NULL; }
+ ;
+
+json_table_nested_columns:
+ NESTED path_opt Sconst
+ json_table_columns_clause
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->columns = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | NESTED path_opt Sconst AS name
+ json_table_columns_clause
+ {
+ JsonTableColumn *n = makeNode(JsonTableColumn);
+
+ n->coltype = JTC_NESTED;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->columns = $6;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+path_opt:
+ PATH { }
+ | /* EMPTY */ { }
+ ;
+
+json_table_plan_clause_opt:
+ PLAN '(' json_table_plan ')' { $$ = $3; }
+ | PLAN DEFAULT '(' json_table_default_plan_choices ')'
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_DEFAULT;
+ n->join_type = $4;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | /* EMPTY */ { $$ = NULL; }
+ ;
+
+json_table_plan:
+ json_table_plan_simple
+ | json_table_plan_parent_child
+ | json_table_plan_sibling
+ ;
+
+json_table_plan_simple:
+ name
+ {
+ JsonTablePlan *n = makeNode(JsonTablePlan);
+
+ n->plan_type = JSTP_SIMPLE;
+ n->pathname = $1;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_table_plan_primary:
+ json_table_plan_simple { $$ = $1; }
+ | '(' json_table_plan ')'
+ {
+ castNode(JsonTablePlan, $2)->location = @1;
+ $$ = $2;
+ }
+ ;
+
+json_table_plan_outer:
+ json_table_plan_simple OUTER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_OUTER, $1, $3, @1); }
+ ;
+
+json_table_plan_inner:
+ json_table_plan_simple INNER_P json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_INNER, $1, $3, @1); }
+ ;
+
+json_table_plan_parent_child:
+ json_table_plan_outer
+ | json_table_plan_inner
+ ;
+
+json_table_plan_union:
+ json_table_plan_primary UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ | json_table_plan_union UNION json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_UNION, $1, $3, @1); }
+ ;
+
+json_table_plan_cross:
+ json_table_plan_primary CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ | json_table_plan_cross CROSS json_table_plan_primary
+ { $$ = makeJsonTableJoinedPlan(JSTPJ_CROSS, $1, $3, @1); }
+ ;
+
+json_table_plan_sibling:
+ json_table_plan_union
+ | json_table_plan_cross
+ ;
+
+json_table_default_plan_choices:
+ json_table_default_plan_inner_outer { $$ = $1 | JSTPJ_UNION; }
+ | json_table_default_plan_union_cross { $$ = $1 | JSTPJ_OUTER; }
+ | json_table_default_plan_inner_outer ','
+ json_table_default_plan_union_cross { $$ = $1 | $3; }
+ | json_table_default_plan_union_cross ','
+ json_table_default_plan_inner_outer { $$ = $1 | $3; }
+ ;
+
+json_table_default_plan_inner_outer:
+ INNER_P { $$ = JSTPJ_INNER; }
+ | OUTER_P { $$ = JSTPJ_OUTER; }
+ ;
+
+json_table_default_plan_union_cross:
+ UNION { $$ = JSTPJ_UNION; }
+ | CROSS { $$ = JSTPJ_CROSS; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -17405,6 +17699,7 @@ unreserved_keyword:
| MOVE
| NAME_P
| NAMES
+ | NESTED
| NEW
| NEXT
| NFC
@@ -17439,6 +17734,8 @@ unreserved_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
+ | PLAN
| PLANS
| POLICY
| PRECEDING
@@ -17603,6 +17900,7 @@ col_name_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| LEAST
| NATIONAL
@@ -17971,6 +18269,7 @@ bare_label_keyword:
| JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_TABLE
| JSON_VALUE
| KEEP
| KEY
@@ -18010,6 +18309,7 @@ bare_label_keyword:
| NATIONAL
| NATURAL
| NCHAR
+ | NESTED
| NEW
| NEXT
| NFC
@@ -18054,7 +18354,9 @@ bare_label_keyword:
| PARTITION
| PASSING
| PASSWORD
+ | PATH
| PLACING
+ | PLAN
| PLANS
| POLICY
| POSITION
diff --git a/src/backend/parser/meson.build b/src/backend/parser/meson.build
index 94517a7f7d..0f0e4ac7ca 100644
--- a/src/backend/parser/meson.build
+++ b/src/backend/parser/meson.build
@@ -10,6 +10,7 @@ backend_sources += files(
'parse_enr.c',
'parse_expr.c',
'parse_func.c',
+ 'parse_jsontable.c',
'parse_merge.c',
'parse_node.c',
'parse_oper.c',
diff --git a/src/backend/parser/parse_clause.c b/src/backend/parser/parse_clause.c
index 334b9b42bd..affd812619 100644
--- a/src/backend/parser/parse_clause.c
+++ b/src/backend/parser/parse_clause.c
@@ -697,7 +697,11 @@ transformRangeTableFunc(ParseState *pstate, RangeTableFunc *rtf)
char **names;
int colno;
- /* Currently only XMLTABLE is supported */
+ /*
+ * Currently we only support XMLTABLE here. See transformJsonTable() for
+ * JSON_TABLE support.
+ */
+ tf->functype = TFT_XMLTABLE;
constructName = "XMLTABLE";
docType = XMLOID;
@@ -1104,13 +1108,17 @@ transformFromClauseItem(ParseState *pstate, Node *n,
rtr->rtindex = nsitem->p_rtindex;
return (Node *) rtr;
}
- else if (IsA(n, RangeTableFunc))
+ else if (IsA(n, RangeTableFunc) || IsA(n, JsonTable))
{
/* table function is like a plain relation */
RangeTblRef *rtr;
ParseNamespaceItem *nsitem;
- nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ if (IsA(n, RangeTableFunc))
+ nsitem = transformRangeTableFunc(pstate, (RangeTableFunc *) n);
+ else
+ nsitem = transformJsonTable(pstate, (JsonTable *) n);
+
*top_nsitem = nsitem;
*namespace = list_make1(nsitem);
rtr = makeNode(RangeTblRef);
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64571562c5..11d0f5f5c9 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4351,6 +4351,19 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
JSON_BEHAVIOR_NULL,
jsexpr->returning);
break;
+
+ case JSON_TABLE_OP:
+ func_name = "JSON_TABLE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->returning = makeNode(JsonReturning);
+ jsexpr->returning->format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+ jsexpr->returning->typid = exprType(jsexpr->formatted_expr);
+ jsexpr->returning->typmod = -1;
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_EMPTY,
+ jsexpr->returning);
+ break;
}
Assert(jsexpr != NULL && jsexpr->formatted_expr != NULL);
diff --git a/src/backend/parser/parse_jsontable.c b/src/backend/parser/parse_jsontable.c
new file mode 100644
index 0000000000..36d888b301
--- /dev/null
+++ b/src/backend/parser/parse_jsontable.c
@@ -0,0 +1,760 @@
+/*-------------------------------------------------------------------------
+ *
+ * parse_jsontable.c
+ * parsing of JSON_TABLE
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/backend/parser/parse_jsontable.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "catalog/pg_collation.h"
+#include "catalog/pg_type.h"
+#include "miscadmin.h"
+#include "nodes/makefuncs.h"
+#include "nodes/nodeFuncs.h"
+#include "optimizer/optimizer.h"
+#include "parser/parse_clause.h"
+#include "parser/parse_collate.h"
+#include "parser/parse_expr.h"
+#include "parser/parse_relation.h"
+#include "parser/parse_type.h"
+#include "utils/builtins.h"
+#include "utils/json.h"
+#include "utils/lsyscache.h"
+
+/* Context for JSON_TABLE transformation */
+typedef struct JsonTableParseContext
+{
+ ParseState *pstate; /* parsing state */
+ JsonTable *table; /* untransformed node */
+ TableFunc *tablefunc; /* transformed node */
+ List *pathNames; /* list of all path and columns names */
+ int pathNameId; /* path name id counter */
+ Oid contextItemTypid; /* type oid of context item (json/jsonb) */
+} JsonTableParseContext;
+
+static JsonTableParent *transformJsonTableColumns(JsonTableParseContext * cxt,
+ JsonTablePlan *plan,
+ List *columns,
+ char *pathSpec,
+ char **pathName,
+ int location);
+
+static Node *
+makeStringConst(char *str, int location)
+{
+ A_Const *n = makeNode(A_Const);
+
+ n->val.node.type = T_String;
+ n->val.sval.sval = str;
+ n->location = location;
+
+ return (Node *) n;
+}
+
+/*
+ * Transform JSON_TABLE column
+ * - regular column into JSON_VALUE()
+ * - FORMAT JSON column into JSON_QUERY()
+ * - EXISTS column into JSON_EXISTS()
+ */
+static Node *
+transformJsonTableColumn(JsonTableColumn *jtc, Node *contextItemExpr,
+ List *passingArgs, bool errorOnError)
+{
+ JsonFuncExpr *jfexpr = makeNode(JsonFuncExpr);
+ JsonCommon *common = makeNode(JsonCommon);
+ JsonOutput *output = makeNode(JsonOutput);
+ char *pathspec;
+ JsonFormat *default_format;
+
+ jfexpr->op =
+ jtc->coltype == JTC_REGULAR ? JSON_VALUE_OP :
+ jtc->coltype == JTC_EXISTS ? JSON_EXISTS_OP : JSON_QUERY_OP;
+ jfexpr->common = common;
+ jfexpr->output = output;
+ jfexpr->behavior = jtc->behavior;
+ if ((jfexpr->behavior == NULL || jfexpr->behavior->on_error == NULL) &&
+ errorOnError)
+ {
+ JsonBehavior *on_error = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL,
+ -1);
+
+ if (jfexpr->behavior == NULL)
+ jfexpr->behavior = makeJsonBehaviorClause(NULL, on_error, -1);
+ else
+ jfexpr->behavior->on_error = on_error;
+ }
+ jfexpr->quotes = jtc->quotes;
+ jfexpr->wrapper = jtc->wrapper;
+ jfexpr->location = jtc->location;
+
+ output->typeName = jtc->typeName;
+ output->returning = makeNode(JsonReturning);
+ output->returning->format = jtc->format;
+
+ default_format = makeJsonFormat(JS_FORMAT_DEFAULT, JS_ENC_DEFAULT, -1);
+
+ common->pathname = NULL;
+ /* formatted_expr will be set later in transformJsonValueExpr(). */
+ common->expr = makeJsonValueExpr((Expr *) contextItemExpr, NULL,
+ default_format);
+ common->passing = passingArgs;
+
+ if (jtc->pathspec)
+ pathspec = jtc->pathspec;
+ else
+ {
+ /* Construct default path as '$."column_name"' */
+ StringInfoData path;
+
+ initStringInfo(&path);
+
+ appendStringInfoString(&path, "$.");
+ escape_json(&path, jtc->name);
+
+ pathspec = path.data;
+ }
+
+ common->pathspec = makeStringConst(pathspec, -1);
+
+ return (Node *) jfexpr;
+}
+
+static bool
+isJsonTablePathNameDuplicate(JsonTableParseContext * cxt, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, cxt->pathNames)
+ {
+ if (!strcmp(pathname, (const char *) lfirst(lc)))
+ return true;
+ }
+
+ return false;
+}
+
+/* Register the column name in the path name list. */
+static void
+registerJsonTableColumn(JsonTableParseContext * cxt, char *colname)
+{
+ if (isJsonTablePathNameDuplicate(cxt, colname))
+ ereport(ERROR,
+ (errcode(ERRCODE_DUPLICATE_ALIAS),
+ errmsg("duplicate JSON_TABLE column name: %s", colname),
+ errhint("JSON_TABLE column names must be distinct from one another.")));
+
+ cxt->pathNames = lappend(cxt->pathNames, colname);
+}
+
+/* Recursively register all nested column names in the path name list. */
+static void
+registerAllJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ if (jtc->pathname)
+ registerJsonTableColumn(cxt, jtc->pathname);
+
+ registerAllJsonTableColumns(cxt, jtc->columns);
+ }
+ else
+ {
+ registerJsonTableColumn(cxt, jtc->name);
+ }
+ }
+}
+
+/* Generate a new unique JSON_TABLE path name. */
+static char *
+generateJsonTablePathName(JsonTableParseContext * cxt)
+{
+ char namebuf[32];
+ char *name = namebuf;
+
+ do
+ {
+ snprintf(namebuf, sizeof(namebuf), "json_table_path_%d",
+ ++cxt->pathNameId);
+ } while (isJsonTablePathNameDuplicate(cxt, name));
+
+ name = pstrdup(name);
+ cxt->pathNames = lappend(cxt->pathNames, name);
+
+ return name;
+}
+
+/* Collect sibling path names from plan to the specified list. */
+static void
+collectSiblingPathsInJsonTablePlan(JsonTablePlan *plan, List **paths)
+{
+ if (plan->plan_type == JSTP_SIMPLE)
+ *paths = lappend(*paths, plan->pathname);
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ *paths = lappend(*paths, plan->plan1->pathname);
+ }
+ else if (plan->join_type == JSTPJ_CROSS ||
+ plan->join_type == JSTPJ_UNION)
+ {
+ collectSiblingPathsInJsonTablePlan(plan->plan1, paths);
+ collectSiblingPathsInJsonTablePlan(plan->plan2, paths);
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE join type %d",
+ plan->join_type);
+ }
+}
+
+/*
+ * Validate child JSON_TABLE plan by checking that:
+ * - all nested columns have path names specified
+ * - all nested columns have corresponding node in the sibling plan
+ * - plan does not contain duplicate or extra nodes
+ */
+static void
+validateJsonTableChildPlan(ParseState *pstate, JsonTablePlan *plan,
+ List *columns)
+{
+ ListCell *lc1;
+ List *siblings = NIL;
+ int nchildren = 0;
+
+ if (plan)
+ collectSiblingPathsInJsonTablePlan(plan, &siblings);
+
+ foreach(lc1, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc1));
+
+ if (jtc->coltype == JTC_NESTED)
+ {
+ ListCell *lc2;
+ bool found = false;
+
+ if (!jtc->pathname)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("nested JSON_TABLE columns must contain an explicit AS pathname specification if an explicit PLAN clause is used"),
+ parser_errposition(pstate, jtc->location)));
+
+ /* find nested path name in the list of sibling path names */
+ foreach(lc2, siblings)
+ {
+ if ((found = !strcmp(jtc->pathname, lfirst(lc2))))
+ break;
+ }
+
+ if (!found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node for nested path %s was not found in plan.", jtc->pathname),
+ parser_errposition(pstate, jtc->location)));
+
+ nchildren++;
+ }
+ }
+
+ if (list_length(siblings) > nchildren)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Plan node contains some extra or duplicate sibling nodes."),
+ parser_errposition(pstate, plan ? plan->location : -1)));
+}
+
+static JsonTableColumn *
+findNestedJsonTableColumn(List *columns, const char *pathname)
+{
+ ListCell *lc;
+
+ foreach(lc, columns)
+ {
+ JsonTableColumn *jtc = castNode(JsonTableColumn, lfirst(lc));
+
+ if (jtc->coltype == JTC_NESTED &&
+ jtc->pathname &&
+ !strcmp(jtc->pathname, pathname))
+ return jtc;
+ }
+
+ return NULL;
+}
+
+static Node *
+transformNestedJsonTableColumn(JsonTableParseContext * cxt, JsonTableColumn *jtc,
+ JsonTablePlan *plan)
+{
+ JsonTableParent *node;
+ char *pathname = jtc->pathname;
+
+ node = transformJsonTableColumns(cxt, plan, jtc->columns, jtc->pathspec,
+ &pathname, jtc->location);
+
+ return (Node *) node;
+}
+
+static Node *
+makeJsonTableSiblingJoin(bool cross, Node *lnode, Node *rnode)
+{
+ JsonTableSibling *join = makeNode(JsonTableSibling);
+
+ join->larg = lnode;
+ join->rarg = rnode;
+ join->cross = cross;
+
+ return (Node *) join;
+}
+
+/*
+ * Recursively transform child JSON_TABLE plan.
+ *
+ * Default plan is transformed into a cross/union join of its nested columns.
+ * Simple and outer/inner plans are transformed into a JsonTableParent by
+ * finding and transforming corresponding nested column.
+ * Sibling plans are recursively transformed into a JsonTableSibling.
+ */
+static Node *
+transformJsonTableChildPlan(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns)
+{
+ JsonTableColumn *jtc = NULL;
+
+ if (!plan || plan->plan_type == JSTP_DEFAULT)
+ {
+ /* unspecified or default plan */
+ Node *res = NULL;
+ ListCell *lc;
+ bool cross = plan && (plan->join_type & JSTPJ_CROSS);
+
+ /* transform all nested columns into cross/union join */
+ foreach(lc, columns)
+ {
+ JsonTableColumn *col = castNode(JsonTableColumn, lfirst(lc));
+ Node *node;
+
+ if (col->coltype != JTC_NESTED)
+ continue;
+
+ node = transformNestedJsonTableColumn(cxt, col, plan);
+
+ /* join transformed node with previous sibling nodes */
+ res = res ? makeJsonTableSiblingJoin(cross, res, node) : node;
+ }
+
+ return res;
+ }
+ else if (plan->plan_type == JSTP_SIMPLE)
+ {
+ jtc = findNestedJsonTableColumn(columns, plan->pathname);
+ }
+ else if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type == JSTPJ_INNER ||
+ plan->join_type == JSTPJ_OUTER)
+ {
+ Assert(plan->plan1->plan_type == JSTP_SIMPLE);
+ jtc = findNestedJsonTableColumn(columns, plan->plan1->pathname);
+ }
+ else
+ {
+ Node *node1 = transformJsonTableChildPlan(cxt, plan->plan1,
+ columns);
+ Node *node2 = transformJsonTableChildPlan(cxt, plan->plan2,
+ columns);
+
+ return makeJsonTableSiblingJoin(plan->join_type == JSTPJ_CROSS,
+ node1, node2);
+ }
+ }
+ else
+ elog(ERROR, "invalid JSON_TABLE plan type %d", plan->plan_type);
+
+ if (!jtc)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name was %s not found in nested columns list.",
+ plan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ return transformNestedJsonTableColumn(cxt, jtc, plan);
+}
+
+/* Check whether type is json/jsonb, array, or record. */
+static bool
+typeIsComposite(Oid typid)
+{
+ char typtype;
+
+ if (typid == JSONOID ||
+ typid == JSONBOID ||
+ typid == RECORDOID ||
+ type_is_array(typid))
+ return true;
+
+ typtype = get_typtype(typid);
+
+ if (typtype == TYPTYPE_COMPOSITE)
+ return true;
+
+ if (typtype == TYPTYPE_DOMAIN)
+ return typeIsComposite(getBaseType(typid));
+
+ return false;
+}
+
+/* Append transformed non-nested JSON_TABLE columns to the TableFunc node */
+static void
+appendJsonTableColumns(JsonTableParseContext * cxt, List *columns)
+{
+ ListCell *col;
+ ParseState *pstate = cxt->pstate;
+ JsonTable *jt = cxt->table;
+ TableFunc *tf = cxt->tablefunc;
+ bool ordinality_found = false;
+ JsonBehavior *on_error = jt->behavior ? jt->behavior->on_error : NULL;
+ bool errorOnError = on_error &&
+ on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ foreach(col, columns)
+ {
+ JsonTableColumn *rawc = castNode(JsonTableColumn, lfirst(col));
+ Oid typid;
+ int32 typmod;
+ Node *colexpr;
+
+ if (rawc->name)
+ {
+ /* make sure column names are unique */
+ ListCell *colname;
+
+ foreach(colname, tf->colnames)
+ if (!strcmp((const char *) colname, rawc->name))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("column name \"%s\" is not unique",
+ rawc->name),
+ parser_errposition(pstate, rawc->location)));
+
+ tf->colnames = lappend(tf->colnames,
+ makeString(pstrdup(rawc->name)));
+ }
+
+ /*
+ * Determine the type and typmod for the new column. FOR ORDINALITY
+ * columns are INTEGER by standard; the others are user-specified.
+ */
+ switch (rawc->coltype)
+ {
+ case JTC_FOR_ORDINALITY:
+ if (ordinality_found)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use more than one FOR ORDINALITY column"),
+ parser_errposition(pstate, rawc->location)));
+ ordinality_found = true;
+ colexpr = NULL;
+ typid = INT4OID;
+ typmod = -1;
+ break;
+
+ case JTC_REGULAR:
+ typenameTypeIdAndMod(pstate, rawc->typeName, &typid, &typmod);
+
+ /*
+ * Use implicit FORMAT JSON for composite types (arrays and
+ * records).
+ *
+ * For scalar columns, require an explicit FORMAT JSON clause
+ * to get the specified WRAPPER and QUOTES behavior.
+ */
+ if (typeIsComposite(typid))
+ rawc->coltype = JTC_FORMATTED;
+ else if (rawc->wrapper != JSW_NONE)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use WITH WRAPPER clause for scalar columns without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+ else if (rawc->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use OMIT/KEEP QUOTES clause for scalar columns without also specifying FORMAT clause"),
+ parser_errposition(pstate, rawc->location)));
+
+ /* FALLTHROUGH */
+ case JTC_EXISTS:
+ case JTC_FORMATTED:
+ {
+ Node *je;
+ CaseTestExpr *param = makeNode(CaseTestExpr);
+
+ param->collation = InvalidOid;
+ param->typeId = cxt->contextItemTypid;
+ param->typeMod = -1;
+
+ je = transformJsonTableColumn(rawc, (Node *) param,
+ NIL, errorOnError);
+
+ colexpr = transformExpr(pstate, je, EXPR_KIND_FROM_FUNCTION);
+ assign_expr_collations(pstate, colexpr);
+
+ typid = exprType(colexpr);
+ typmod = exprTypmod(colexpr);
+ break;
+ }
+
+ case JTC_NESTED:
+ continue;
+
+ default:
+ elog(ERROR, "unknown JSON_TABLE column type: %d", rawc->coltype);
+ break;
+ }
+
+ tf->coltypes = lappend_oid(tf->coltypes, typid);
+ tf->coltypmods = lappend_int(tf->coltypmods, typmod);
+ tf->colcollations = lappend_oid(tf->colcollations, get_typcollation(typid));
+ tf->colvalexprs = lappend(tf->colvalexprs, colexpr);
+ }
+}
+
+/*
+ * Create transformed JSON_TABLE parent plan node by appending all non-nested
+ * columns to the TableFunc node and remembering their indices in the
+ * colvalexprs list.
+ */
+static JsonTableParent *
+makeParentJsonTableNode(JsonTableParseContext * cxt, char *pathSpec, char *pathName,
+ List *columns)
+{
+ JsonTableParent *node = makeNode(JsonTableParent);
+ JsonBehavior *on_error = cxt->table->behavior ?
+ cxt->table->behavior->on_error : NULL;
+
+ node->path = makeNode(JsonTablePath);
+ node->path->value = makeConst(JSONPATHOID, -1, InvalidOid, -1,
+ DirectFunctionCall1(jsonpath_in,
+ CStringGetDatum(pathSpec)),
+ false, false);
+ if (pathName)
+ node->path->name = pstrdup(pathName);
+
+ /* save start of column range */
+ node->colMin = list_length(cxt->tablefunc->colvalexprs);
+
+ appendJsonTableColumns(cxt, columns);
+
+ /* save end of column range */
+ node->colMax = list_length(cxt->tablefunc->colvalexprs) - 1;
+
+ node->errorOnError = on_error && on_error->btype == JSON_BEHAVIOR_ERROR;
+
+ return node;
+}
+
+static JsonTableParent *
+transformJsonTableColumns(JsonTableParseContext * cxt, JsonTablePlan *plan,
+ List *columns, char *pathSpec, char **pathName,
+ int location)
+{
+ JsonTableParent *node;
+ JsonTablePlan *childPlan;
+ bool defaultPlan = !plan || plan->plan_type == JSTP_DEFAULT;
+
+ if (!*pathName)
+ {
+ if (cxt->table->plan)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE expression"),
+ errdetail("JSON_TABLE columns must contain "
+ "explicit AS pathname specification if "
+ "explicit PLAN clause is used"),
+ parser_errposition(cxt->pstate, location)));
+
+ *pathName = generateJsonTablePathName(cxt);
+ }
+
+ if (defaultPlan)
+ childPlan = plan;
+ else
+ {
+ /* validate parent and child plans */
+ JsonTablePlan *parentPlan;
+
+ if (plan->plan_type == JSTP_JOINED)
+ {
+ if (plan->join_type != JSTPJ_INNER &&
+ plan->join_type != JSTPJ_OUTER)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Expected INNER or OUTER JSON_TABLE plan node."),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ parentPlan = plan->plan1;
+ childPlan = plan->plan2;
+
+ Assert(parentPlan->plan_type != JSTP_JOINED);
+ Assert(parentPlan->pathname);
+ }
+ else
+ {
+ parentPlan = plan;
+ childPlan = NULL;
+ }
+
+ if (strcmp(parentPlan->pathname, *pathName))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid JSON_TABLE plan"),
+ errdetail("Path name mismatch: expected %s but %s is given.",
+ *pathName, parentPlan->pathname),
+ parser_errposition(cxt->pstate, plan->location)));
+
+ validateJsonTableChildPlan(cxt->pstate, childPlan, columns);
+ }
+
+ /* transform only non-nested columns */
+ node = makeParentJsonTableNode(cxt, pathSpec, *pathName, columns);
+
+ if (childPlan || defaultPlan)
+ {
+ /* transform recursively nested columns */
+ node->child = transformJsonTableChildPlan(cxt, childPlan, columns);
+ if (node->child)
+ node->outerJoin = !plan || (plan->join_type & JSTPJ_OUTER);
+ /* else: default plan case, no children found */
+ }
+
+ return node;
+}
+
+/*
+ * transformJsonTable -
+ * Transform a raw JsonTable into TableFunc.
+ *
+ * Transform the document-generating expression, the row-generating expression,
+ * the column-generating expressions, and the default value expressions.
+ */
+ParseNamespaceItem *
+transformJsonTable(ParseState *pstate, JsonTable *jt)
+{
+ JsonTableParseContext cxt;
+ TableFunc *tf = makeNode(TableFunc);
+ JsonFuncExpr *jfe = makeNode(JsonFuncExpr);
+ JsonExpr *je;
+ JsonTablePlan *plan = jt->plan;
+ JsonCommon *jscommon;
+ char *rootPathName = jt->common->pathname;
+ char *rootPath;
+ bool is_lateral;
+
+ cxt.pstate = pstate;
+ cxt.table = jt;
+ cxt.tablefunc = tf;
+ cxt.pathNames = NIL;
+ cxt.pathNameId = 0;
+
+ if (rootPathName)
+ registerJsonTableColumn(&cxt, rootPathName);
+
+ registerAllJsonTableColumns(&cxt, jt->columns);
+
+#if 0 /* XXX it' unclear from the standard whether
+ * root path name is mandatory or not */
+ if (plan && plan->plan_type != JSTP_DEFAULT && !rootPathName)
+ {
+ /* Assign root path name and create corresponding plan node */
+ JsonTablePlan *rootNode = makeNode(JsonTablePlan);
+ JsonTablePlan *rootPlan = (JsonTablePlan *)
+ makeJsonTableJoinedPlan(JSTPJ_OUTER, (Node *) rootNode,
+ (Node *) plan, jt->location);
+
+ rootPathName = generateJsonTablePathName(&cxt);
+
+ rootNode->plan_type = JSTP_SIMPLE;
+ rootNode->pathname = rootPathName;
+
+ plan = rootPlan;
+ }
+#endif
+
+ jscommon = copyObject(jt->common);
+ jscommon->pathspec = makeStringConst(pstrdup("$"), -1);
+
+ jfe->op = JSON_TABLE_OP;
+ jfe->common = jscommon;
+ jfe->behavior = jt->behavior;
+ jfe->location = jt->common->location;
+
+ /*
+ * We make lateral_only names of this level visible, whether or not the
+ * RangeTableFunc is explicitly marked LATERAL. This is needed for SQL
+ * spec compliance and seems useful on convenience grounds for all
+ * functions in FROM.
+ *
+ * (LATERAL can't nest within a single pstate level, so we don't need
+ * save/restore logic here.)
+ */
+ Assert(!pstate->p_lateral_active);
+ pstate->p_lateral_active = true;
+
+ tf->functype = TFT_JSON_TABLE;
+ tf->docexpr = transformExpr(pstate, (Node *) jfe, EXPR_KIND_FROM_FUNCTION);
+
+ cxt.contextItemTypid = exprType(tf->docexpr);
+
+ if (!IsA(jt->common->pathspec, A_Const) ||
+ castNode(A_Const, jt->common->pathspec)->val.node.type != T_String)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("only string constants supported in JSON_TABLE path specification"),
+ parser_errposition(pstate,
+ exprLocation(jt->common->pathspec))));
+
+ rootPath = castNode(A_Const, jt->common->pathspec)->val.sval.sval;
+
+ tf->plan = (Node *) transformJsonTableColumns(&cxt, plan, jt->columns,
+ rootPath, &rootPathName,
+ jt->common->location);
+
+ /* Also save a copy of the PASSING arguments in the TableFunc node. */
+ je = (JsonExpr *) tf->docexpr;
+ tf->passingvalexprs = copyObject(je->passing_values);
+
+ tf->ordinalitycol = -1; /* undefine ordinality column number */
+ tf->location = jt->location;
+
+ pstate->p_lateral_active = false;
+
+ /*
+ * Mark the RTE as LATERAL if the user said LATERAL explicitly, or if
+ * there are any lateral cross-references in it.
+ */
+ is_lateral = jt->lateral || contain_vars_of_level((Node *) tf, 0);
+
+ return addRangeTableEntryForTableFunc(pstate,
+ tf, jt->alias, is_lateral, true);
+}
diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c
index 864ea9b0d5..3ac7e113f0 100644
--- a/src/backend/parser/parse_relation.c
+++ b/src/backend/parser/parse_relation.c
@@ -2073,7 +2073,8 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
Assert(list_length(tf->coltypmods) == list_length(tf->colnames));
Assert(list_length(tf->colcollations) == list_length(tf->colnames));
- refname = alias ? alias->aliasname : pstrdup("xmltable");
+ refname = alias ? alias->aliasname :
+ pstrdup(tf->functype == TFT_XMLTABLE ? "xmltable" : "json_table");
rte->rtekind = RTE_TABLEFUNC;
rte->relid = InvalidOid;
@@ -2096,7 +2097,7 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
errmsg("%s function has %d columns available but %d columns specified",
- "XMLTABLE",
+ tf->functype == TFT_XMLTABLE ? "XMLTABLE" : "JSON_TABLE",
list_length(tf->colnames), numaliases)));
rte->eref = eref;
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 8f81624f97..ca7228ea41 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -2002,6 +2002,9 @@ FigureColnameInternal(Node *node, char **name)
case JSON_VALUE_OP:
*name = "json_value";
return 2;
+ case JSON_TABLE_OP:
+ *name = "json_table";
+ return 2;
}
break;
default:
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 6c5602c64d..79632e3dfd 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -61,10 +61,12 @@
#include "catalog/pg_collation.h"
#include "catalog/pg_type.h"
+#include "executor/execExpr.h"
#include "funcapi.h"
#include "lib/stringinfo.h"
#include "miscadmin.h"
#include "nodes/miscnodes.h"
+#include "nodes/nodeFuncs.h"
#include "regex/regex.h"
#include "utils/builtins.h"
#include "utils/date.h"
@@ -75,6 +77,8 @@
#include "utils/guc.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
+#include "utils/lsyscache.h"
+#include "utils/memutils.h"
#include "utils/timestamp.h"
#include "utils/varlena.h"
@@ -156,6 +160,61 @@ typedef struct JsonValueListIterator
ListCell *next;
} JsonValueListIterator;
+/* Structures for JSON_TABLE execution */
+
+typedef enum JsonTablePlanStateType
+{
+ JSON_TABLE_SCAN_STATE = 0,
+ JSON_TABLE_JOIN_STATE
+} JsonTablePlanStateType;
+
+typedef struct JsonTablePlanState
+{
+ JsonTablePlanStateType type;
+
+ struct JsonTablePlanState *parent;
+ struct JsonTablePlanState *nested;
+} JsonTablePlanState;
+
+typedef struct JsonTableScanState
+{
+ JsonTablePlanState plan;
+
+ MemoryContext mcxt;
+ JsonPath *path;
+ List *args;
+ JsonValueList found;
+ JsonValueListIterator iter;
+ Datum current;
+ int ordinal;
+ bool currentIsNull;
+ bool outerJoin;
+ bool errorOnError;
+ bool advanceNested;
+ bool reset;
+} JsonTableScanState;
+
+typedef struct JsonTableJoinState
+{
+ JsonTablePlanState plan;
+
+ JsonTablePlanState *left;
+ JsonTablePlanState *right;
+ bool cross;
+ bool advanceRight;
+} JsonTableJoinState;
+
+/* random number to identify JsonTableExecContext */
+#define JSON_TABLE_EXEC_CONTEXT_MAGIC 418352867
+
+typedef struct JsonTableExecContext
+{
+ int magic;
+ JsonTableScanState **colexprscans;
+ JsonTableScanState *root;
+ bool empty;
+} JsonTableExecContext;
+
/* strict/lax flags is decomposed into four [un]wrap/error flags */
#define jspStrictAbsenseOfErrors(cxt) (!(cxt)->laxMode)
#define jspAutoUnwrap(cxt) ((cxt)->laxMode)
@@ -248,6 +307,7 @@ static JsonPathExecResult getArrayIndex(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonbValue *jb, int32 *index);
static JsonBaseObjectInfo setBaseObject(JsonPathExecContext *cxt,
JsonbValue *jbv, int32 id);
+static void JsonValueListClear(JsonValueList *jvl);
static void JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv);
static int JsonValueListLength(const JsonValueList *jvl);
static bool JsonValueListIsEmpty(JsonValueList *jvl);
@@ -265,6 +325,14 @@ static JsonbValue *wrapItemsInArray(const JsonValueList *items);
static int compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
bool useTz, bool *cast_error);
+
+static JsonTablePlanState * JsonTableInitPlanState(JsonTableExecContext * cxt,
+ Node *plan,
+ JsonTablePlanState * parent);
+static bool JsonTablePlanNextRow(JsonTablePlanState * state);
+static bool JsonTableScanNextRow(JsonTableScanState *scan);
+
+
/****************** User interface to JsonPath executor ********************/
/*
@@ -2518,6 +2586,13 @@ setBaseObject(JsonPathExecContext *cxt, JsonbValue *jbv, int32 id)
return baseObject;
}
+static void
+JsonValueListClear(JsonValueList *jvl)
+{
+ jvl->singleton = NULL;
+ jvl->list = NULL;
+}
+
static void
JsonValueListAppend(JsonValueList *jvl, JsonbValue *jbv)
{
@@ -3123,3 +3198,471 @@ JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
}
}
+
+/************************ JSON_TABLE functions ***************************/
+
+/*
+ * Returns private data from executor state. Ensure validity by check with
+ * MAGIC number.
+ */
+static inline JsonTableExecContext *
+GetJsonTableExecContext(TableFuncScanState *state, const char *fname)
+{
+ JsonTableExecContext *result;
+
+ if (!IsA(state, TableFuncScanState))
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+ result = (JsonTableExecContext *) state->opaque;
+ if (result->magic != JSON_TABLE_EXEC_CONTEXT_MAGIC)
+ elog(ERROR, "%s called with invalid TableFuncScanState", fname);
+
+ return result;
+}
+
+/* Recursively initialize JSON_TABLE scan / join state */
+static JsonTableJoinState *
+JsonTableInitJoinState(JsonTableExecContext * cxt, JsonTableSibling *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTableJoinState *join = palloc0(sizeof(*join));
+
+ join->plan.type = JSON_TABLE_JOIN_STATE;
+ /* parent and nested not set. */
+
+ join->cross = plan->cross;
+ join->left = JsonTableInitPlanState(cxt, plan->larg, parent);
+ join->right = JsonTableInitPlanState(cxt, plan->rarg, parent);
+
+ return join;
+}
+
+static JsonTableScanState *
+JsonTableInitScanState(JsonTableExecContext * cxt, JsonTableParent *plan,
+ JsonTablePlanState * parent,
+ List *args, MemoryContext mcxt)
+{
+ JsonTableScanState *scan = palloc0(sizeof(*scan));
+ int i;
+
+ scan->plan.type = JSON_TABLE_SCAN_STATE;
+ scan->plan.parent = parent;
+
+ scan->outerJoin = plan->outerJoin;
+ scan->errorOnError = plan->errorOnError;
+ scan->path = DatumGetJsonPathP(plan->path->value->constvalue);
+ scan->args = args;
+ scan->mcxt = AllocSetContextCreate(mcxt, "JsonTableExecContext",
+ ALLOCSET_DEFAULT_SIZES);
+
+ /*
+ * Set after settings scan->args and scan->mcxt, because the recursive
+ * call wants to use those values.
+ */
+ scan->plan.nested = plan->child ?
+ JsonTableInitPlanState(cxt, plan->child, (JsonTablePlanState *) scan) :
+ NULL;
+
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+
+ for (i = plan->colMin; i <= plan->colMax; i++)
+ cxt->colexprscans[i] = scan;
+
+ return scan;
+}
+
+/* Recursively initialize JSON_TABLE scan state */
+static JsonTablePlanState *
+JsonTableInitPlanState(JsonTableExecContext * cxt, Node *plan,
+ JsonTablePlanState * parent)
+{
+ JsonTablePlanState *state;
+
+ if (IsA(plan, JsonTableSibling))
+ {
+ JsonTableSibling *join = castNode(JsonTableSibling, plan);
+
+ state = (JsonTablePlanState *)
+ JsonTableInitJoinState(cxt, join, parent);
+ }
+ else
+ {
+ JsonTableParent *scan = castNode(JsonTableParent, plan);
+ JsonTableScanState *parent_scan = (JsonTableScanState *) parent;
+
+ Assert(parent_scan);
+ state = (JsonTablePlanState *)
+ JsonTableInitScanState(cxt, scan, parent, parent_scan->args,
+ parent_scan->mcxt);
+ }
+
+ return state;
+}
+
+/*
+ * JsonTableInitOpaque
+ * Fill in TableFuncScanState->opaque for JsonTable processor
+ */
+static void
+JsonTableInitOpaque(TableFuncScanState *state, int natts)
+{
+ JsonTableExecContext *cxt;
+ PlanState *ps = &state->ss.ps;
+ TableFuncScan *tfs = castNode(TableFuncScan, ps->plan);
+ TableFunc *tf = tfs->tablefunc;
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+ JsonExpr *je = castNode(JsonExpr, tf->docexpr);
+ List *args = NIL;
+
+ cxt = palloc0(sizeof(JsonTableExecContext));
+ cxt->magic = JSON_TABLE_EXEC_CONTEXT_MAGIC;
+
+ if (state->passingvalexprs)
+ {
+ ListCell *exprlc;
+ ListCell *namelc;
+
+ Assert(list_length(state->passingvalexprs) ==
+ list_length(je->passing_names));
+ forboth(exprlc, state->passingvalexprs,
+ namelc, je->passing_names)
+ {
+ ExprState *state = lfirst_node(ExprState, exprlc);
+ String *name = lfirst_node(String, namelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = pstrdup(name->sval);
+ var->typid = exprType((Node *) state->expr);
+ var->typmod = exprTypmod((Node *) state->expr);
+
+ /*
+ * Evaluate the expression and save the value to be returned by
+ * GetJsonPathVar().
+ */
+ var->value = ExecEvalExpr(state, ps->ps_ExprContext,
+ &var->isnull);
+
+ args = lappend(args, var);
+ }
+ }
+
+ cxt->colexprscans = palloc(sizeof(JsonTableScanState *) *
+ list_length(tf->colvalexprs));
+
+ cxt->root = JsonTableInitScanState(cxt, root, NULL, args,
+ CurrentMemoryContext);
+ state->opaque = cxt;
+}
+
+/* Reset scan iterator to the beginning of the item list */
+static void
+JsonTableRescan(JsonTableScanState *scan)
+{
+ JsonValueListInitIterator(&scan->found, &scan->iter);
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ scan->advanceNested = false;
+ scan->ordinal = 0;
+}
+
+/* Reset context item of a scan, execute JSON path and reset a scan */
+static void
+JsonTableResetContextItem(JsonTableScanState *scan, Datum item)
+{
+ MemoryContext oldcxt;
+ JsonPathExecResult res;
+ Jsonb *js = (Jsonb *) DatumGetJsonbP(item);
+
+ JsonValueListClear(&scan->found);
+
+ MemoryContextResetOnly(scan->mcxt);
+
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+
+ res = executeJsonPath(scan->path, scan->args, GetJsonPathVar, js,
+ scan->errorOnError, &scan->found,
+ false /* FIXME */ );
+
+ MemoryContextSwitchTo(oldcxt);
+
+ if (jperIsError(res))
+ {
+ Assert(!scan->errorOnError);
+ JsonValueListClear(&scan->found); /* EMPTY ON ERROR case */
+ }
+
+ JsonTableRescan(scan);
+}
+
+/*
+ * JsonTableSetDocument
+ * Install the input document
+ */
+static void
+JsonTableSetDocument(TableFuncScanState *state, Datum value)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableSetDocument");
+
+ JsonTableResetContextItem(cxt->root, value);
+}
+
+/* Recursively reset scan and its child nodes */
+static void
+JsonTableRescanRecursive(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTableRescanRecursive(join->left);
+ JsonTableRescanRecursive(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan = (JsonTableScanState *) state;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ JsonTableRescan(scan);
+ if (scan->plan.nested)
+ JsonTableRescanRecursive(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a cross/union joined scan.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTablePlanNextRow(JsonTablePlanState * state)
+{
+ JsonTableJoinState *join;
+
+ if (state->type == JSON_TABLE_SCAN_STATE)
+ return JsonTableScanNextRow((JsonTableScanState *) state);
+
+ join = (JsonTableJoinState *) state;
+ if (join->advanceRight)
+ {
+ /* fetch next inner row */
+ if (JsonTablePlanNextRow(join->right))
+ return true;
+
+ /* inner rows are exhausted */
+ if (join->cross)
+ join->advanceRight = false; /* next outer row */
+ else
+ return false; /* end of scan */
+ }
+
+ while (!join->advanceRight)
+ {
+ /* fetch next outer row */
+ bool left = JsonTablePlanNextRow(join->left);
+
+ if (join->cross)
+ {
+ if (!left)
+ return false; /* end of scan */
+
+ JsonTableRescanRecursive(join->right);
+
+ if (!JsonTablePlanNextRow(join->right))
+ continue; /* next outer row */
+
+ join->advanceRight = true; /* next inner row */
+ }
+ else if (!left)
+ {
+ if (!JsonTablePlanNextRow(join->right))
+ return false; /* end of scan */
+
+ join->advanceRight = true; /* next inner row */
+ }
+
+ break;
+ }
+
+ return true;
+}
+
+/* Recursively set 'reset' flag of scan and its child nodes */
+static void
+JsonTablePlanReset(JsonTablePlanState * state)
+{
+ if (state->type == JSON_TABLE_JOIN_STATE)
+ {
+ JsonTableJoinState *join = (JsonTableJoinState *) state;
+
+ JsonTablePlanReset(join->left);
+ JsonTablePlanReset(join->right);
+ join->advanceRight = false;
+ }
+ else
+ {
+ JsonTableScanState *scan;
+
+ Assert(state->type == JSON_TABLE_SCAN_STATE);
+ scan = (JsonTableScanState *) state;
+ scan->reset = true;
+ scan->advanceNested = false;
+
+ if (scan->plan.nested)
+ JsonTablePlanReset(scan->plan.nested);
+ }
+}
+
+/*
+ * Fetch next row from a simple scan with outer/inner joined nested subscans.
+ *
+ * Returns false at the end of a scan, true otherwise.
+ */
+static bool
+JsonTableScanNextRow(JsonTableScanState *scan)
+{
+ /* reset context item if requested */
+ if (scan->reset)
+ {
+ JsonTableScanState *parent_scan =
+ (JsonTableScanState *) scan->plan.parent;
+
+ Assert(parent_scan && !parent_scan->currentIsNull);
+ JsonTableResetContextItem(scan, parent_scan->current);
+ scan->reset = false;
+ }
+
+ if (scan->advanceNested)
+ {
+ /* fetch next nested row */
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested)
+ return true;
+ }
+
+ for (;;)
+ {
+ /* fetch next row */
+ JsonbValue *jbv = JsonValueListNext(&scan->found, &scan->iter);
+ MemoryContext oldcxt;
+
+ if (!jbv)
+ {
+ scan->current = PointerGetDatum(NULL);
+ scan->currentIsNull = true;
+ return false; /* end of scan */
+ }
+
+ /* set current row item */
+ oldcxt = MemoryContextSwitchTo(scan->mcxt);
+ scan->current = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ scan->currentIsNull = false;
+ MemoryContextSwitchTo(oldcxt);
+
+ scan->ordinal++;
+
+ if (!scan->plan.nested)
+ break;
+
+ JsonTablePlanReset(scan->plan.nested);
+
+ scan->advanceNested = JsonTablePlanNextRow(scan->plan.nested);
+
+ if (scan->advanceNested || scan->outerJoin)
+ break;
+ }
+
+ return true;
+}
+
+/*
+ * JsonTableFetchRow
+ * Prepare the next "current" tuple for upcoming GetValue calls.
+ * Returns FALSE if the row-filter expression returned no more rows.
+ */
+static bool
+JsonTableFetchRow(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableFetchRow");
+
+ if (cxt->empty)
+ return false;
+
+ return JsonTableScanNextRow(cxt->root);
+}
+
+/*
+ * JsonTableGetValue
+ * Return the value for column number 'colnum' for the current row.
+ *
+ * This leaks memory, so be sure to reset often the context in which it's
+ * called.
+ */
+static Datum
+JsonTableGetValue(TableFuncScanState *state, int colnum,
+ Oid typid, int32 typmod, bool *isnull)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableGetValue");
+ ExprContext *econtext = state->ss.ps.ps_ExprContext;
+ ExprState *estate = list_nth(state->colvalexprs, colnum);
+ JsonTableScanState *scan = cxt->colexprscans[colnum];
+ Datum result;
+
+ if (scan->currentIsNull) /* NULL from outer/union join */
+ {
+ result = (Datum) 0;
+ *isnull = true;
+ }
+ else if (estate) /* regular column */
+ {
+ Datum saved_caseValue = econtext->caseValue_datum;
+ bool saved_caseIsNull = econtext->caseValue_isNull;
+
+ /* Pass the value for CaseTestExpr that may be present in colexpr */
+ econtext->caseValue_datum = scan->current;
+ econtext->caseValue_isNull = false;
+
+ result = ExecEvalExpr(estate, econtext, isnull);
+
+ econtext->caseValue_datum = saved_caseValue;
+ econtext->caseValue_isNull = saved_caseIsNull;
+ }
+ else
+ {
+ result = Int32GetDatum(scan->ordinal); /* ordinality column */
+ *isnull = false;
+ }
+
+ return result;
+}
+
+/*
+ * JsonTableDestroyOpaque
+ */
+static void
+JsonTableDestroyOpaque(TableFuncScanState *state)
+{
+ JsonTableExecContext *cxt =
+ GetJsonTableExecContext(state, "JsonTableDestroyOpaque");
+
+ /* not valid anymore */
+ cxt->magic = 0;
+
+ state->opaque = NULL;
+}
+
+const TableFuncRoutine JsonbTableRoutine =
+{
+ JsonTableInitOpaque,
+ JsonTableSetDocument,
+ NULL,
+ NULL,
+ NULL,
+ JsonTableFetchRow,
+ JsonTableGetValue,
+ JsonTableDestroyOpaque
+};
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index 342a2cf6aa..ca13107536 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -522,6 +522,8 @@ static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
static void get_json_path_spec(Node *path_spec, deparse_context *context,
bool showimplicit);
+static void get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8627,7 +8629,8 @@ get_json_behavior(JsonBehavior *behavior, deparse_context *context,
/*
* get_json_expr_options
*
- * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS and
+ * JSON_TABLE columns.
*/
static void
get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
@@ -9875,6 +9878,9 @@ get_rule_expr(Node *node, deparse_context *context,
case JSON_VALUE_OP:
appendStringInfoString(buf, "JSON_VALUE(");
break;
+ default:
+ elog(ERROR, "unexpected JsonExpr type: %d", jexpr->op);
+ break;
}
get_rule_expr(jexpr->formatted_expr, context, showimplicit);
@@ -11241,16 +11247,14 @@ get_sublink_expr(SubLink *sublink, deparse_context *context)
/* ----------
- * get_tablefunc - Parse back a table function
+ * get_xmltable - Parse back a XMLTABLE function
* ----------
*/
static void
-get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+get_xmltable(TableFunc *tf, deparse_context *context, bool showimplicit)
{
StringInfo buf = context->buf;
- /* XMLTABLE is the only existing implementation. */
-
appendStringInfoString(buf, "XMLTABLE(");
if (tf->ns_uris != NIL)
@@ -11341,6 +11345,271 @@ get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
appendStringInfoChar(buf, ')');
}
+/*
+ * get_json_nested_columns - Parse back nested JSON_TABLE columns
+ */
+static void
+get_json_table_nested_columns(TableFunc *tf, Node *node,
+ deparse_context *context, bool showimplicit,
+ bool needcomma)
+{
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_nested_columns(tf, n->larg, context, showimplicit,
+ needcomma);
+ get_json_table_nested_columns(tf, n->rarg, context, showimplicit, true);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ if (needcomma)
+ appendStringInfoChar(context->buf, ',');
+
+ appendStringInfoChar(context->buf, ' ');
+ appendContextKeyword(context, "NESTED PATH ", 0, 0, 0);
+ get_const_expr(n->path->value, context, -1);
+ appendStringInfo(context->buf, " AS %s", quote_identifier(n->path->name));
+ get_json_table_columns(tf, n, context, showimplicit);
+ }
+}
+
+/*
+ * get_json_table_plan - Parse back a JSON_TABLE plan
+ */
+static void
+get_json_table_plan(TableFunc *tf, Node *node, deparse_context *context,
+ bool parenthesize)
+{
+ if (parenthesize)
+ appendStringInfoChar(context->buf, '(');
+
+ if (IsA(node, JsonTableSibling))
+ {
+ JsonTableSibling *n = (JsonTableSibling *) node;
+
+ get_json_table_plan(tf, n->larg, context,
+ IsA(n->larg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->larg)->child);
+
+ appendStringInfoString(context->buf, n->cross ? " CROSS " : " UNION ");
+
+ get_json_table_plan(tf, n->rarg, context,
+ IsA(n->rarg, JsonTableSibling) ||
+ castNode(JsonTableParent, n->rarg)->child);
+ }
+ else
+ {
+ JsonTableParent *n = castNode(JsonTableParent, node);
+
+ appendStringInfoString(context->buf, quote_identifier(n->path->name));
+
+ if (n->child)
+ {
+ appendStringInfoString(context->buf,
+ n->outerJoin ? " OUTER " : " INNER ");
+ get_json_table_plan(tf, n->child, context,
+ IsA(n->child, JsonTableSibling));
+ }
+ }
+
+ if (parenthesize)
+ appendStringInfoChar(context->buf, ')');
+}
+
+/*
+ * get_json_table_columns - Parse back JSON_TABLE columns
+ */
+static void
+get_json_table_columns(TableFunc *tf, JsonTableParent *node,
+ deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ ListCell *lc_colname;
+ ListCell *lc_coltype;
+ ListCell *lc_coltypmod;
+ ListCell *lc_colvalexpr;
+ int colnum = 0;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "COLUMNS (", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forfour(lc_colname, tf->colnames,
+ lc_coltype, tf->coltypes,
+ lc_coltypmod, tf->coltypmods,
+ lc_colvalexpr, tf->colvalexprs)
+ {
+ char *colname = strVal(lfirst(lc_colname));
+ JsonExpr *colexpr;
+ Oid typid;
+ int32 typmod;
+ bool ordinality;
+ JsonBehaviorType default_behavior;
+
+ typid = lfirst_oid(lc_coltype);
+ typmod = lfirst_int(lc_coltypmod);
+ colexpr = castNode(JsonExpr, lfirst(lc_colvalexpr));
+
+ if (colnum < node->colMin)
+ {
+ colnum++;
+ continue;
+ }
+
+ if (colnum > node->colMax)
+ break;
+
+ if (colnum > node->colMin)
+ appendStringInfoString(buf, ", ");
+
+ colnum++;
+
+ ordinality = !colexpr;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ appendStringInfo(buf, "%s %s", quote_identifier(colname),
+ ordinality ? "FOR ORDINALITY" :
+ format_type_with_typemod(typid, typmod));
+ if (ordinality)
+ continue;
+
+ if (colexpr->op == JSON_EXISTS_OP)
+ {
+ appendStringInfoString(buf, " EXISTS");
+ default_behavior = JSON_BEHAVIOR_FALSE;
+ }
+ else
+ {
+ if (colexpr->op == JSON_QUERY_OP)
+ {
+ char typcategory;
+ bool typispreferred;
+
+ get_type_category_preferred(typid, &typcategory, &typispreferred);
+
+ if (typcategory == TYPCATEGORY_STRING)
+ appendStringInfoString(buf,
+ colexpr->format->format_type == JS_FORMAT_JSONB ?
+ " FORMAT JSONB" : " FORMAT JSON");
+ }
+
+ default_behavior = JSON_BEHAVIOR_NULL;
+ }
+
+ if (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR)
+ default_behavior = JSON_BEHAVIOR_ERROR;
+
+ appendStringInfoString(buf, " PATH ");
+
+ get_json_path_spec(colexpr->path_spec, context, showimplicit);
+
+ get_json_expr_options(colexpr, context, default_behavior);
+ }
+
+ if (node->child)
+ get_json_table_nested_columns(tf, node->child, context, showimplicit,
+ node->colMax >= node->colMin);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_json_table - Parse back a JSON_TABLE function
+ * ----------
+ */
+static void
+get_json_table(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ StringInfo buf = context->buf;
+ JsonExpr *jexpr = castNode(JsonExpr, tf->docexpr);
+ JsonTableParent *root = castNode(JsonTableParent, tf->plan);
+
+ appendStringInfoString(buf, "JSON_TABLE(");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_const_expr(root->path->value, context, -1);
+
+ appendStringInfo(buf, " AS %s", quote_identifier(root->path->name));
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PASSING ", 0, 0, 0);
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel += PRETTYINDENT_VAR;
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ appendContextKeyword(context, "", 0, 0, 0);
+
+ get_rule_expr((Node *) lfirst(lc2), context, false);
+ appendStringInfo(buf, " AS %s",
+ quote_identifier((lfirst_node(String, lc1))->sval)
+ );
+ }
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+ }
+
+ get_json_table_columns(tf, root, context, showimplicit);
+
+ appendStringInfoChar(buf, ' ');
+ appendContextKeyword(context, "PLAN ", 0, 0, 0);
+ get_json_table_plan(tf, (Node *) root, context, true);
+
+ if (jexpr->on_error->btype != JSON_BEHAVIOR_EMPTY)
+ get_json_behavior(jexpr->on_error, context, "ERROR");
+
+ if (PRETTY_INDENT(context))
+ context->indentLevel -= PRETTYINDENT_VAR;
+
+ appendContextKeyword(context, ")", 0, 0, 0);
+}
+
+/* ----------
+ * get_tablefunc - Parse back a table function
+ * ----------
+ */
+static void
+get_tablefunc(TableFunc *tf, deparse_context *context, bool showimplicit)
+{
+ /* XMLTABLE and JSON_TABLE are the only existing implementations. */
+
+ if (tf->functype == TFT_XMLTABLE)
+ get_xmltable(tf, context, showimplicit);
+ else if (tf->functype == TFT_JSON_TABLE)
+ get_json_table(tf, context, showimplicit);
+}
+
/* ----------
* get_from_clause - Parse back a FROM clause
*
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 6a7118d300..2fa0328977 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -1882,6 +1882,8 @@ typedef struct TableFuncScanState
ExprState *rowexpr; /* state for row-generating expression */
List *colexprs; /* state for column-generating expression */
List *coldefexprs; /* state for column default expressions */
+ List *colvalexprs; /* state for column value expression */
+ List *passingvalexprs; /* state for PASSING argument expression */
List *ns_names; /* same as TableFunc.ns_names */
List *ns_uris; /* list of states of namespace URI exprs */
Bitmapset *notnulls; /* nullability flag for each output column */
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 783e82ac92..79c486eceb 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -116,6 +116,8 @@ extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int loc
extern JsonBehaviorClause *makeJsonBehaviorClause(JsonBehavior *on_empty,
JsonBehavior *on_error,
int location);
+extern Node *makeJsonTableJoinedPlan(JsonTablePlanJoinType type,
+ Node *plan1, Node *plan2, int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ce6893fb91..7ed60a7d2c 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1721,6 +1721,19 @@ typedef struct JsonBehaviorClause
*/
typedef char *JsonPathSpec;
+/*
+ * JsonTableColumnType -
+ * enumeration of JSON_TABLE column types
+ */
+typedef enum JsonTableColumnType
+{
+ JTC_FOR_ORDINALITY,
+ JTC_REGULAR,
+ JTC_EXISTS,
+ JTC_FORMATTED,
+ JTC_NESTED,
+} JsonTableColumnType;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1773,6 +1786,82 @@ typedef struct JsonFuncExpr
int location; /* token location, or -1 if unknown */
} JsonFuncExpr;
+/*
+ * JsonTableColumn -
+ * untransformed representation of JSON_TABLE column
+ */
+typedef struct JsonTableColumn
+{
+ NodeTag type;
+ JsonTableColumnType coltype; /* column type */
+ char *name; /* column name */
+ TypeName *typeName; /* column type name */
+ char *pathspec; /* path specification, if any */
+ char *pathname; /* path name, if any */
+ JsonFormat *format; /* JSON format clause, if specified */
+ JsonWrapper wrapper; /* WRAPPER behavior for formatted columns */
+ JsonQuotes quotes; /* omit or keep quotes on scalar strings? */
+ List *columns; /* nested columns */
+ JsonBehaviorClause *behavior; /* ON ERROR / EMPTY behavior, if specified */
+ int location; /* token location, or -1 if unknown */
+} JsonTableColumn;
+
+/*
+ * JsonTablePlanType -
+ * flags for JSON_TABLE plan node types representation
+ */
+typedef enum JsonTablePlanType
+{
+ JSTP_DEFAULT,
+ JSTP_SIMPLE,
+ JSTP_JOINED,
+} JsonTablePlanType;
+
+/*
+ * JsonTablePlanJoinType -
+ * flags for JSON_TABLE join types representation
+ */
+typedef enum JsonTablePlanJoinType
+{
+ JSTPJ_INNER = 0x01,
+ JSTPJ_OUTER = 0x02,
+ JSTPJ_CROSS = 0x04,
+ JSTPJ_UNION = 0x08,
+} JsonTablePlanJoinType;
+
+typedef struct JsonTablePlan JsonTablePlan;
+
+/*
+ * JsonTablePlan -
+ * untransformed representation of JSON_TABLE plan node
+ */
+struct JsonTablePlan
+{
+ NodeTag type;
+ JsonTablePlanType plan_type; /* plan type */
+ JsonTablePlanJoinType join_type; /* join type (for joined plan only) */
+ JsonTablePlan *plan1; /* first joined plan */
+ JsonTablePlan *plan2; /* second joined plan */
+ char *pathname; /* path name (for simple plan only) */
+ int location; /* token location, or -1 if unknown */
+};
+
+/*
+ * JsonTable -
+ * untransformed representation of JSON_TABLE
+ */
+typedef struct JsonTable
+{
+ NodeTag type;
+ JsonCommon *common; /* common JSON path syntax fields */
+ List *columns; /* list of JsonTableColumn */
+ JsonTablePlan *plan; /* join plan, if specified */
+ JsonBehaviorClause *behavior; /* ON ERROR behavior, if specified */
+ Alias *alias; /* table alias in FROM clause */
+ bool lateral; /* does it have LATERAL prefix? */
+ int location; /* token location, or -1 if unknown */
+} JsonTable;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index 3f75c4608e..4fe392bddb 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -94,8 +94,14 @@ typedef struct RangeVar
int location;
} RangeVar;
+typedef enum TableFuncType
+{
+ TFT_XMLTABLE,
+ TFT_JSON_TABLE
+} TableFuncType;
+
/*
- * TableFunc - node for a table function, such as XMLTABLE.
+ * TableFunc - node for a table function, such as XMLTABLE or JSON_TABLE.
*
* Entries in the ns_names list are either String nodes containing
* literal namespace names, or NULL pointers to represent DEFAULT.
@@ -103,6 +109,8 @@ typedef struct RangeVar
typedef struct TableFunc
{
NodeTag type;
+ /* XMLTABLE or JSON_TABLE */
+ TableFuncType functype;
/* list of namespace URI expressions */
List *ns_uris pg_node_attr(query_jumble_ignore);
/* list of namespace names or NULL */
@@ -123,8 +131,14 @@ typedef struct TableFunc
List *colexprs;
/* list of column default expressions */
List *coldefexprs pg_node_attr(query_jumble_ignore);
+ /* list of column value expressions */
+ List *colvalexprs pg_node_attr(query_jumble_ignore);
+ /* list of PASSING argument expressions */
+ List *passingvalexprs pg_node_attr(query_jumble_ignore);
/* nullability flag for each output column */
Bitmapset *notnulls pg_node_attr(query_jumble_ignore);
+ /* JSON_TABLE plan */
+ Node *plan pg_node_attr(query_jumble_ignore);
/* counts from 0; -1 if none specified */
int ordinalitycol pg_node_attr(query_jumble_ignore);
/* token location, or -1 if unknown */
@@ -1560,7 +1574,8 @@ typedef enum JsonExprOp
{
JSON_VALUE_OP, /* JSON_VALUE() */
JSON_QUERY_OP, /* JSON_QUERY() */
- JSON_EXISTS_OP /* JSON_EXISTS() */
+ JSON_EXISTS_OP, /* JSON_EXISTS() */
+ JSON_TABLE_OP /* JSON_TABLE() */
} JsonExprOp;
/*
@@ -1795,6 +1810,48 @@ typedef struct JsonExpr
int location; /* token location, or -1 if unknown */
} JsonExpr;
+/*
+ * JsonTablePath
+ * A JSON path expression to be computed as part of evaluating
+ * a JSON_TABLE plan node
+ */
+typedef struct JsonTablePath
+{
+ NodeTag type;
+
+ Const *value;
+ char *name;
+} JsonTablePath;
+
+/*
+ * JsonTableParent -
+ * transformed representation of parent JSON_TABLE plan node
+ */
+typedef struct JsonTableParent
+{
+ NodeTag type;
+ JsonTablePath *path;
+ Node *child; /* nested columns, if any */
+ bool outerJoin; /* outer or inner join for nested columns? */
+ int colMin; /* min column index in the resulting column
+ * list */
+ int colMax; /* max column index in the resulting column
+ * list */
+ bool errorOnError; /* ERROR/EMPTY ON ERROR behavior */
+} JsonTableParent;
+
+/*
+ * JsonTableSibling -
+ * transformed representation of joined sibling JSON_TABLE plan node
+ */
+typedef struct JsonTableSibling
+{
+ NodeTag type;
+ Node *larg; /* left join node */
+ Node *rarg; /* right join node */
+ bool cross; /* cross or union join? */
+} JsonTableSibling;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 0954d9fc7b..f88a6c9ac6 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -242,6 +242,7 @@ PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_table", JSON_TABLE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -284,6 +285,7 @@ PG_KEYWORD("names", NAMES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("national", NATIONAL, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("natural", NATURAL, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("nchar", NCHAR, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("nested", NESTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("new", NEW, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("next", NEXT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("nfc", NFC, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -334,7 +336,9 @@ PG_KEYWORD("partial", PARTIAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("partition", PARTITION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("passing", PASSING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("password", PASSWORD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("path", PATH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("placing", PLACING, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("plan", PLAN, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("plans", PLANS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("policy", POLICY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("position", POSITION, COL_NAME_KEYWORD, BARE_LABEL)
diff --git a/src/include/parser/parse_clause.h b/src/include/parser/parse_clause.h
index 8e26482db4..7152a8ef48 100644
--- a/src/include/parser/parse_clause.h
+++ b/src/include/parser/parse_clause.h
@@ -51,4 +51,7 @@ extern List *addTargetToSortList(ParseState *pstate, TargetEntry *tle,
extern Index assignSortGroupRef(TargetEntry *tle, List *tlist);
extern bool targetIsInSortList(TargetEntry *tle, Oid sortop, List *sortList);
+/* functions in parse_jsontable.c */
+extern ParseNamespaceItem *transformJsonTable(ParseState *pstate, JsonTable *jt);
+
#endif /* PARSE_CLAUSE_H */
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index 5a37133847..7eb0ccf4e4 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -15,6 +15,7 @@
#define JSONPATH_H
#include "fmgr.h"
+#include "executor/tablefunc.h"
#include "nodes/pg_list.h"
#include "nodes/primnodes.h"
#include "utils/jsonb.h"
@@ -288,4 +289,6 @@ extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
bool *error, List *vars);
+extern PGDLLIMPORT const TableFuncRoutine JsonbTableRoutine;
+
#endif
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
index 8b87580752..995f267404 100644
--- a/src/test/regress/expected/json_sqljson.out
+++ b/src/test/regress/expected/json_sqljson.out
@@ -16,3 +16,9 @@ ERROR: JSON_QUERY() is not yet implemented for the json type
LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
^
HINT: Try casting the argument to jsonb
+-- JSON_TABLE
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
+ERROR: JSON_TABLE() is not yet implemented for the json type
+LINE 1: SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo ...
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
index b467971ed9..3f171b90e7 100644
--- a/src/test/regress/expected/jsonb_sqljson.out
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -1095,3 +1095,1189 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.dateti
ERROR: functions in index expression must be marked IMMUTABLE
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+-- JSON_TABLE
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+ERROR: syntax error at or near "("
+LINE 1: SELECT JSON_TABLE('[]', '$');
+ ^
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ERROR: syntax error at or near ")"
+LINE 1: SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+ ^
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+ERROR: JSON_TABLE function has 1 columns available but 2 columns specified
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+ foo
+-----
+(0 rows)
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+ item | foo
+------+-----
+ 123 |
+(1 row)
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | int | text | char(4) | bool | numeric | domain | js | jb
+---------------------------------------------------------------------------------------+----+-----+---------+---------+------+---------+---------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [] | | | | | | | | |
+ {} | 1 | | | | | | | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1 | 1.23 | 1.23 | | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | 2 | 2 | 2 | | 2 | 2 | "2" | "2"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | aaaaaaa | aaaa | | | aaaaaaa | "aaaaaaa" | "aaaaaaa"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | foo | foo | | | | "foo" | "foo"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | | | | | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | 0 | false | fals | f | | false | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | 1 | true | true | t | | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | | | | | | | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | [1,2] | [1,2 | | | [1,2] | "[1,2]" | "[1,2]"
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | "str" | "str | | | "str" | "\"str\"" | "\"str\""
+(14 rows)
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+ js | id | jst | jsc | jsv | jsb | jsbq
+---------------------------------------------------------------------------------------+----+--------------+------+------+--------------+--------------
+ 1 | 1 | 1 | 1 | 1 | 1 | 1
+ [] | | | | | |
+ {} | 1 | {} | {} | {} | {} | {}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | 1 | 1 | 1 | 1 | 1
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | 1.23 | 1.23 | 1.23 | 1.23 | 1.23
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | "2" | "2" | "2" | "2" | 2
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | "aaaaaaa" | "aaa | "aaa | "aaaaaaa" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | "foo" | "foo | "foo | "foo" |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | null | null | null | null | null
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | false | fals | fals | false | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | true | true | true | true | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | {"aaa": 123} | {"aa | {"aa | {"aaa": 123} | {"aaa": 123}
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | "[1,2]" | "[1, | "[1, | "[1,2]" | [1, 2]
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | "\"str\"" | "\"s | "\"s | "\"str\"" | "str"
+(14 rows)
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+ js | id | exists1 | exists2 | exists3 | exists4
+---------------------------------------------------------------------------------------+----+---------+---------+---------+---------
+ 1 | 1 | f | 0 | | false
+ [] | | | | |
+ {} | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | t | 1 | 1 | true
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | f | 0 | | false
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | f | 0 | | false
+(14 rows)
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+ js | id | aaa | aaa1 | js2 | jsb2w | jsb2q | ia | ta | jba
+---------------------------------------------------------------------------------------+----+-----+------+--------------+----------------+--------------+----+----+-----
+ 1 | 1 | | | 1 | [1] | 1 | | |
+ [] | | | | | | | | |
+ {} | 1 | | | {} | [{}] | {} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 1 | | | 1 | [1] | 1 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 2 | | | 1.23 | [1.23] | 1.23 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 3 | | | "2" | ["2"] | 2 | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 4 | | | "aaaaaaa" | ["aaaaaaa"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 5 | | | "foo" | ["foo"] | | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 6 | | | null | [null] | null | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 7 | | | false | [false] | false | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 8 | | | true | [true] | true | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 9 | 123 | 123 | {"aaa": 123} | [{"aaa": 123}] | {"aaa": 123} | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 10 | | | "[1,2]" | ["[1,2]"] | [1, 2] | | |
+ [1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""] | 11 | | | "\"str\"" | ["\"str\""] | "str" | | |
+(14 rows)
+
+-- JSON_TABLE: Test backward parsing
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+\sv jsonb_table_view
+CREATE OR REPLACE VIEW public.jsonb_table_view AS
+ SELECT id,
+ "int",
+ text,
+ "char(4)",
+ bool,
+ "numeric",
+ domain,
+ js,
+ jb,
+ jst,
+ jsc,
+ jsv,
+ jsb,
+ jsbq,
+ aaa,
+ aaa1,
+ exists1,
+ exists2,
+ exists3,
+ js2,
+ jsb2w,
+ jsb2q,
+ ia,
+ ta,
+ jba,
+ a1,
+ b1,
+ a11,
+ a21,
+ a22
+ FROM JSON_TABLE(
+ 'null'::jsonb, '$[*]' AS json_table_path_1
+ PASSING
+ 1 + 2 AS a,
+ '"foo"'::json AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" integer PATH '$',
+ text text PATH '$',
+ "char(4)" character(4) PATH '$',
+ bool boolean PATH '$',
+ "numeric" numeric PATH '$',
+ domain jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc character(4) FORMAT JSON PATH '$',
+ jsv character varying(4) FORMAT JSON PATH '$',
+ jsb jsonb PATH '$',
+ jsbq jsonb PATH '$' OMIT QUOTES,
+ aaa integer PATH '$."aaa"',
+ aaa1 integer PATH '$."aaa"',
+ exists1 boolean EXISTS PATH '$."aaa"',
+ exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR,
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia integer[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+ NESTED PATH '$[1]' AS p1
+ COLUMNS (
+ a1 integer PATH '$."a1"',
+ b1 text PATH '$."b1"',
+ NESTED PATH '$[*]' AS "p1 1"
+ COLUMNS (
+ a11 text PATH '$."a11"'
+ )
+ ),
+ NESTED PATH '$[2]' AS p2
+ COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1"
+ COLUMNS (
+ a21 text PATH '$."a21"'
+ ),
+ NESTED PATH '$[*]' AS p22
+ COLUMNS (
+ a22 text PATH '$."a22"'
+ )
+ )
+ )
+ PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22))))
+ )
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+ QUERY PLAN
+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ Table Function Scan on "json_table"
+ Output: "json_table".id, "json_table"."int", "json_table".text, "json_table"."char(4)", "json_table".bool, "json_table"."numeric", "json_table".domain, "json_table".js, "json_table".jb, "json_table".jst, "json_table".jsc, "json_table".jsv, "json_table".jsb, "json_table".jsbq, "json_table".aaa, "json_table".aaa1, "json_table".exists1, "json_table".exists2, "json_table".exists3, "json_table".js2, "json_table".jsb2w, "json_table".jsb2q, "json_table".ia, "json_table".ta, "json_table".jba, "json_table".a1, "json_table".b1, "json_table".a11, "json_table".a21, "json_table".a22
+ Table Function Call: JSON_TABLE('null'::jsonb, '$[*]' AS json_table_path_1 PASSING 3 AS a, '"foo"'::jsonb AS "b c" COLUMNS (id FOR ORDINALITY, "int" integer PATH '$', text text PATH '$', "char(4)" character(4) PATH '$', bool boolean PATH '$', "numeric" numeric PATH '$', domain jsonb_test_domain PATH '$', js json PATH '$', jb jsonb PATH '$', jst text FORMAT JSON PATH '$', jsc character(4) FORMAT JSON PATH '$', jsv character varying(4) FORMAT JSON PATH '$', jsb jsonb PATH '$', jsbq jsonb PATH '$' OMIT QUOTES, aaa integer PATH '$."aaa"', aaa1 integer PATH '$."aaa"', exists1 boolean EXISTS PATH '$."aaa"', exists2 integer EXISTS PATH '$."aaa"' TRUE ON ERROR, exists3 text EXISTS PATH 'strict $."aaa"' UNKNOWN ON ERROR, js2 json PATH '$', jsb2w jsonb PATH '$' WITH UNCONDITIONAL WRAPPER, jsb2q jsonb PATH '$' OMIT QUOTES, ia integer[] PATH '$', ta text[] PATH '$', jba jsonb[] PATH '$', NESTED PATH '$[1]' AS p1 COLUMNS (a1 integer PATH '$."a1"', b1 text PATH '$."b1"', NESTED PATH '$[*]' AS "p1 1" COLUMNS (a11 text PATH '$."a11"')), NESTED PATH '$[2]' AS p2 COLUMNS ( NESTED PATH '$[*]' AS "p2:1" COLUMNS (a21 text PATH '$."a21"'), NESTED PATH '$[*]' AS p22 COLUMNS (a22 text PATH '$."a22"'))) PLAN (json_table_path_1 OUTER ((p1 OUTER "p1 1") UNION (p2 OUTER ("p2:1" UNION p22)))))
+(3 rows)
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+ERROR: cannot use more than one FOR ORDINALITY column
+LINE 1: ..._TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR OR...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+ id | a
+----+---
+ 1 | 1
+(1 row)
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+ js | a
+-------+---
+ 1 | 1
+ "err" |
+(2 rows)
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+ERROR: invalid input syntax for type integer: "err"
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+ a
+---
+
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+ERROR: no SQL/JSON item
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 2
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+ a
+---
+ 1
+(1 row)
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+ a
+---
+ 0
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to smallint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to bigint
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to real
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 E...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+ a
+-----
+ fal
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to json
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXI...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+ERROR: cannot cast type boolean to jsonb
+LINE 1: ...ELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EX...
+ ^
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+ERROR: cannot use OMIT/KEEP QUOTES clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+ item
+---------
+ "world"
+(1 row)
+
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+ERROR: cannot use WITH WRAPPER clause for scalar columns without also specifying FORMAT clause
+LINE 1: ...T * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text ...
+ ^
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+ item
+-----------
+ ["world"]
+(1 row)
+
+-- JSON_TABLE: nested paths and plans
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb '[]', '$' -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 4: NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: b
+HINT: JSON_TABLE column names must be distinct from one another.
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+ERROR: duplicate JSON_TABLE column name: a
+HINT: JSON_TABLE column names must be distinct from one another.
+-- JSON_TABLE: plan validation
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p1)
+ ^
+DETAIL: Path name mismatch: expected p0 but p1 is given.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 4: NESTED PATH '$' AS p1 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p1 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 UNION p1 UNION p11)
+ ^
+DETAIL: Expected INNER or OUTER JSON_TABLE plan node.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 8: NESTED PATH '$' AS p2 COLUMNS (
+ ^
+DETAIL: Plan node for nested path p2 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 5: NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ ^
+DETAIL: Plan node for nested path p11 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 12: PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+ ^
+DETAIL: Plan node contains some extra or duplicate sibling nodes.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 6: NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ^
+DETAIL: Plan node for nested path p12 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+ERROR: invalid JSON_TABLE plan
+LINE 9: NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ ^
+DETAIL: Plan node for nested path p21 was not found in plan.
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+ bar | foo | baz
+-----+-----+-----
+(0 rows)
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+ERROR: invalid JSON_TABLE expression
+LINE 2: jsonb 'null', 'strict $[*]' -- without root path name
+ ^
+DETAIL: JSON_TABLE columns must contain explicit AS pathname specification if explicit PLAN clause is used
+-- JSON_TABLE: plan execution
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(11 rows)
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+ n | a | c | b
+---+----+----+---
+ 1 | 1 | |
+ 2 | 2 | 10 |
+ 2 | 2 | |
+ 2 | 2 | 20 |
+ 2 | 2 | | 1
+ 2 | 2 | | 2
+ 2 | 2 | | 3
+ 3 | 3 | | 1
+ 3 | 3 | | 2
+ 4 | -1 | | 1
+ 4 | -1 | | 2
+(11 rows)
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 2 | 2 | 1 |
+ 2 | 2 | 2 |
+ 2 | 2 | 3 |
+ 2 | 2 | | 10
+ 2 | 2 | |
+ 2 | 2 | | 20
+ 3 | 3 | 1 |
+ 3 | 3 | 2 |
+ 4 | -1 | 1 |
+ 4 | -1 | 2 |
+(10 rows)
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+---+---+----
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+(9 rows)
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+ n | a | b | c
+---+----+---+----
+ 1 | 1 | |
+ 2 | 2 | 1 | 10
+ 2 | 2 | 1 |
+ 2 | 2 | 1 | 20
+ 2 | 2 | 2 | 10
+ 2 | 2 | 2 |
+ 2 | 2 | 2 | 20
+ 2 | 2 | 3 | 10
+ 2 | 2 | 3 |
+ 2 | 2 | 3 | 20
+ 3 | 3 | |
+ 4 | -1 | |
+(12 rows)
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+ n | a | b | b1 | c | c1 | b
+---+---+--------------+-----+------+----+-----
+ 1 | 1 | [1, 10] | 1 | 1 | | 101
+ 1 | 1 | [1, 10] | 1 | null | | 101
+ 1 | 1 | [1, 10] | 1 | 2 | | 101
+ 1 | 1 | [1, 10] | 10 | 1 | | 110
+ 1 | 1 | [1, 10] | 10 | null | | 110
+ 1 | 1 | [1, 10] | 10 | 2 | | 110
+ 1 | 1 | [2] | 2 | 1 | | 102
+ 1 | 1 | [2] | 2 | null | | 102
+ 1 | 1 | [2] | 2 | 2 | | 102
+ 1 | 1 | [3, 30, 300] | 3 | 1 | | 103
+ 1 | 1 | [3, 30, 300] | 3 | null | | 103
+ 1 | 1 | [3, 30, 300] | 3 | 2 | | 103
+ 1 | 1 | [3, 30, 300] | 30 | 1 | | 130
+ 1 | 1 | [3, 30, 300] | 30 | null | | 130
+ 1 | 1 | [3, 30, 300] | 30 | 2 | | 130
+ 1 | 1 | [3, 30, 300] | 300 | 1 | | 400
+ 1 | 1 | [3, 30, 300] | 300 | null | | 400
+ 1 | 1 | [3, 30, 300] | 300 | 2 | | 400
+ 2 | 2 | | | | |
+ 3 | | | | | |
+(20 rows)
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+ x | y | y | z
+---+---+--------------+---
+ 2 | 1 | [1, 2, 3] | 1
+ 2 | 1 | [1, 2, 3] | 2
+ 2 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [1, 2, 3] | 1
+ 3 | 1 | [1, 2, 3] | 2
+ 3 | 1 | [1, 2, 3] | 3
+ 3 | 1 | [2, 3, 4, 5] | 2
+ 3 | 1 | [2, 3, 4, 5] | 3
+ 3 | 1 | [2, 3, 4, 5] | 4
+ 3 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [1, 2, 3] | 1
+ 4 | 1 | [1, 2, 3] | 2
+ 4 | 1 | [1, 2, 3] | 3
+ 4 | 1 | [2, 3, 4, 5] | 2
+ 4 | 1 | [2, 3, 4, 5] | 3
+ 4 | 1 | [2, 3, 4, 5] | 4
+ 4 | 1 | [2, 3, 4, 5] | 5
+ 4 | 1 | [3, 4, 5, 6] | 3
+ 4 | 1 | [3, 4, 5, 6] | 4
+ 4 | 1 | [3, 4, 5, 6] | 5
+ 4 | 1 | [3, 4, 5, 6] | 6
+ 2 | 2 | [1, 2, 3] | 2
+ 2 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [1, 2, 3] | 2
+ 3 | 2 | [1, 2, 3] | 3
+ 3 | 2 | [2, 3, 4, 5] | 2
+ 3 | 2 | [2, 3, 4, 5] | 3
+ 3 | 2 | [2, 3, 4, 5] | 4
+ 3 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [1, 2, 3] | 2
+ 4 | 2 | [1, 2, 3] | 3
+ 4 | 2 | [2, 3, 4, 5] | 2
+ 4 | 2 | [2, 3, 4, 5] | 3
+ 4 | 2 | [2, 3, 4, 5] | 4
+ 4 | 2 | [2, 3, 4, 5] | 5
+ 4 | 2 | [3, 4, 5, 6] | 3
+ 4 | 2 | [3, 4, 5, 6] | 4
+ 4 | 2 | [3, 4, 5, 6] | 5
+ 4 | 2 | [3, 4, 5, 6] | 6
+ 2 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [1, 2, 3] | 3
+ 3 | 3 | [2, 3, 4, 5] | 3
+ 3 | 3 | [2, 3, 4, 5] | 4
+ 3 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [1, 2, 3] | 3
+ 4 | 3 | [2, 3, 4, 5] | 3
+ 4 | 3 | [2, 3, 4, 5] | 4
+ 4 | 3 | [2, 3, 4, 5] | 5
+ 4 | 3 | [3, 4, 5, 6] | 3
+ 4 | 3 | [3, 4, 5, 6] | 4
+ 4 | 3 | [3, 4, 5, 6] | 5
+ 4 | 3 | [3, 4, 5, 6] | 6
+(52 rows)
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+ERROR: could not find jsonpath variable "x"
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+ json_value
+------------
+ foo
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+ json_query
+------------
+ 123
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+ json_query
+------------
+ [123]
+(1 row)
+
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+ERROR: syntax error at or near " " of jsonpath input
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
+ERROR: only string constants supported in JSON_TABLE path specification
+LINE 1: SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || '...
+ ^
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
index 4f30fa46b9..df4a430d88 100644
--- a/src/test/regress/sql/json_sqljson.sql
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -9,3 +9,7 @@ SELECT JSON_VALUE(NULL FORMAT JSON, '$');
-- JSON_QUERY
SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+
+-- JSON_TABLE
+
+SELECT * FROM JSON_TABLE(NULL FORMAT JSON, '$' COLUMNS (foo text));
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
index b3a32c09f6..801569da31 100644
--- a/src/test/regress/sql/jsonb_sqljson.sql
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -347,3 +347,677 @@ CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime()
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
DROP TABLE test_jsonb_mutability;
+
+-- JSON_TABLE
+
+-- Should fail (JSON_TABLE can be used only in FROM clause)
+SELECT JSON_TABLE('[]', '$');
+
+-- Should fail (no columns)
+SELECT * FROM JSON_TABLE(NULL, '$' COLUMNS ());
+
+SELECT * FROM JSON_TABLE (NULL::jsonb, '$' COLUMNS (v1 timestamp)) AS f (v1, v2);
+
+-- NULL => empty table
+SELECT * FROM JSON_TABLE(NULL::jsonb, '$' COLUMNS (foo int)) bar;
+
+--
+SELECT * FROM JSON_TABLE(jsonb '123', '$'
+ COLUMNS (item int PATH '$', foo int)) bar;
+
+-- JSON_TABLE: basic functionality
+CREATE DOMAIN jsonb_test_domain AS text CHECK (value <> 'foo');
+CREATE TEMP TABLE json_table_test (js) AS
+ (VALUES
+ ('1'),
+ ('[]'),
+ ('{}'),
+ ('[1, 1.23, "2", "aaaaaaa", "foo", null, false, true, {"aaa": 123}, "[1,2]", "\"str\""]')
+ );
+
+-- Regular "unformatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- "formatted" columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES
+ )
+ ) jt
+ ON true;
+
+-- EXISTS columns
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa',
+ exists3 int EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+ exists4 text EXISTS PATH 'strict $.aaa' FALSE ON ERROR
+ )
+ ) jt
+ ON true;
+
+-- Other miscellanous checks
+SELECT *
+FROM json_table_test vals
+ LEFT OUTER JOIN
+ JSON_TABLE(
+ vals.js::jsonb, 'lax $[*]'
+ COLUMNS (
+ id FOR ORDINALITY,
+ aaa int, -- "aaa" has implicit path '$."aaa"'
+ aaa1 int PATH '$.aaa',
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$'
+ )
+ ) jt
+ ON true;
+
+-- JSON_TABLE: Test backward parsing
+
+CREATE VIEW jsonb_table_view AS
+SELECT * FROM
+ JSON_TABLE(
+ jsonb 'null', 'lax $[*]' PASSING 1 + 2 AS a, json '"foo"' AS "b c"
+ COLUMNS (
+ id FOR ORDINALITY,
+ "int" int PATH '$',
+ "text" text PATH '$',
+ "char(4)" char(4) PATH '$',
+ "bool" bool PATH '$',
+ "numeric" numeric PATH '$',
+ "domain" jsonb_test_domain PATH '$',
+ js json PATH '$',
+ jb jsonb PATH '$',
+ jst text FORMAT JSON PATH '$',
+ jsc char(4) FORMAT JSON PATH '$',
+ jsv varchar(4) FORMAT JSON PATH '$',
+ jsb jsonb FORMAT JSON PATH '$',
+ jsbq jsonb FORMAT JSON PATH '$' OMIT QUOTES,
+ aaa int, -- implicit path '$."aaa"',
+ aaa1 int PATH '$.aaa',
+ exists1 bool EXISTS PATH '$.aaa',
+ exists2 int EXISTS PATH '$.aaa' TRUE ON ERROR,
+ exists3 text EXISTS PATH 'strict $.aaa' UNKNOWN ON ERROR,
+
+ js2 json PATH '$',
+ jsb2w jsonb PATH '$' WITH WRAPPER,
+ jsb2q jsonb PATH '$' OMIT QUOTES,
+ ia int[] PATH '$',
+ ta text[] PATH '$',
+ jba jsonb[] PATH '$',
+
+ NESTED PATH '$[1]' AS p1 COLUMNS (
+ a1 int,
+ NESTED PATH '$[*]' AS "p1 1" COLUMNS (
+ a11 text
+ ),
+ b1 text
+ ),
+ NESTED PATH '$[2]' AS p2 COLUMNS (
+ NESTED PATH '$[*]' AS "p2:1" COLUMNS (
+ a21 text
+ ),
+ NESTED PATH '$[*]' AS p22 COLUMNS (
+ a22 text
+ )
+ )
+ )
+ );
+
+\sv jsonb_table_view
+
+EXPLAIN (COSTS OFF, VERBOSE) SELECT * FROM jsonb_table_view;
+
+DROP VIEW jsonb_table_view;
+DROP DOMAIN jsonb_test_domain;
+
+-- JSON_TABLE: only one FOR ORDINALITY columns allowed
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, id2 FOR ORDINALITY, a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (id FOR ORDINALITY, a int PATH '$' ERROR ON EMPTY)) jt;
+
+-- JSON_TABLE: ON EMPTY/ON ERROR behavior
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js),
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$')) jt;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$') ERROR ON ERROR) jt
+ ON true;
+
+SELECT *
+FROM
+ (VALUES ('1'), ('"err"')) vals(js)
+ LEFT OUTER JOIN
+ JSON_TABLE(vals.js::jsonb, '$' COLUMNS (a int PATH '$' ERROR ON ERROR)) jt
+ ON true;
+
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH '$.a' ERROR ON EMPTY)) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'strict $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+SELECT * FROM JSON_TABLE(jsonb '1', '$' COLUMNS (a int PATH 'lax $.a' ERROR ON EMPTY) ERROR ON ERROR) jt;
+
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH '$' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'strict $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int PATH 'lax $.a' DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR)) jt;
+
+-- JSON_TABLE: EXISTS PATH types
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int2 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a int8 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a float4 EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a char(3) EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a json EXISTS PATH '$.a'));
+SELECT * FROM JSON_TABLE(jsonb '"a"', '$' COLUMNS (a jsonb EXISTS PATH '$.a'));
+
+-- JSON_TABLE: WRAPPER/QUOTES clauses on scalar columns don't work without an
+-- explicit FORMAT clause
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' KEEP QUOTES ON SCALAR STRING));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text PATH '$' WITH WRAPPER));
+SELECT * FROM JSON_TABLE(jsonb '"world"', '$' COLUMNS (item text FORMAT JSON PATH '$' WITH WRAPPER));
+
+-- JSON_TABLE: nested paths and plans
+
+-- Should fail (JSON_TABLE columns must contain explicit AS path
+-- specifications if explicit PLAN clause is used)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' -- AS <path name> required here
+ COLUMNS (
+ foo int PATH '$'
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS path1
+ COLUMNS (
+ NESTED PATH '$' COLUMNS ( -- AS <path name> required here
+ foo int PATH '$'
+ )
+ )
+ PLAN DEFAULT (UNION)
+) jt;
+
+-- Should fail (column names must be distinct)
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ a int
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$' AS a
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ b int,
+ NESTED PATH '$' AS b
+ COLUMNS (
+ c int
+ )
+ )
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb '[]', '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ b int
+ ),
+ NESTED PATH '$'
+ COLUMNS (
+ NESTED PATH '$' AS a
+ COLUMNS (
+ c int
+ )
+ )
+ )
+) jt;
+
+-- JSON_TABLE: plan validation
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p1)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER p3)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 UNION p1 UNION p11)
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p13))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER (p1 CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 UNION p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER p11) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', '$[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS p2))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' AS p0
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN (p0 OUTER ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21)))
+) jt;
+
+SELECT * FROM JSON_TABLE(
+ jsonb 'null', 'strict $[*]' -- without root path name
+ COLUMNS (
+ NESTED PATH '$' AS p1 COLUMNS (
+ NESTED PATH '$' AS p11 COLUMNS ( foo int ),
+ NESTED PATH '$' AS p12 COLUMNS ( bar int )
+ ),
+ NESTED PATH '$' AS p2 COLUMNS (
+ NESTED PATH '$' AS p21 COLUMNS ( baz int )
+ )
+ )
+ PLAN ((p1 INNER (p12 CROSS p11)) CROSS (p2 INNER p21))
+) jt;
+
+-- JSON_TABLE: plan execution
+
+CREATE TEMP TABLE jsonb_table_test (js jsonb);
+
+INSERT INTO jsonb_table_test
+VALUES (
+ '[
+ {"a": 1, "b": [], "c": []},
+ {"a": 2, "b": [1, 2, 3], "c": [10, null, 20]},
+ {"a": 3, "b": [1, 2], "c": []},
+ {"x": "4", "b": [1, 2], "c": 123}
+ ]'
+);
+
+-- unspecified plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ ) jt;
+
+-- default plan (outer, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, union)
+ ) jt;
+
+-- specific plan (p outer (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb union pc))
+ ) jt;
+
+-- specific plan (p outer (pc union pb))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pc union pb))
+ ) jt;
+
+-- default plan (inner, union)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (inner)
+ ) jt;
+
+-- specific plan (p inner (pb union pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb union pc))
+ ) jt;
+
+-- default plan (inner, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (cross, inner)
+ ) jt;
+
+-- specific plan (p inner (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p inner (pb cross pc))
+ ) jt;
+
+-- default plan (outer, cross)
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan default (outer, cross)
+ ) jt;
+
+-- specific plan (p outer (pb cross pc))
+select
+ jt.*
+from
+ jsonb_table_test jtt,
+ json_table (
+ jtt.js,'strict $[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on empty,
+ nested path 'strict $.b[*]' as pb columns ( b int path '$' ),
+ nested path 'strict $.c[*]' as pc columns ( c int path '$' )
+ )
+ plan (p outer (pb cross pc))
+ ) jt;
+
+
+select
+ jt.*, b1 + 100 as b
+from
+ json_table (jsonb
+ '[
+ {"a": 1, "b": [[1, 10], [2], [3, 30, 300]], "c": [1, null, 2]},
+ {"a": 2, "b": [10, 20], "c": [1, null, 2]},
+ {"x": "3", "b": [11, 22, 33, 44]}
+ ]',
+ '$[*]' as p
+ columns (
+ n for ordinality,
+ a int path 'lax $.a' default -1 on error,
+ nested path 'strict $.b[*]' as pb columns (
+ b text format json path '$',
+ nested path 'strict $[*]' as pb1 columns (
+ b1 int path '$'
+ )
+ ),
+ nested path 'strict $.c[*]' as pc columns (
+ c text format json path '$',
+ nested path 'strict $[*]' as pc1 columns (
+ c1 int path '$'
+ )
+ )
+ )
+ --plan default(outer, cross)
+ plan(p outer ((pb inner pb1) cross (pc outer pc1)))
+ ) jt;
+
+-- Should succeed (JSON arguments are passed to root and nested paths)
+SELECT *
+FROM
+ generate_series(1, 4) x,
+ generate_series(1, 3) y,
+ JSON_TABLE(jsonb
+ '[[1,2,3],[2,3,4,5],[3,4,5,6]]',
+ 'strict $[*] ? (@[*] < $x)'
+ PASSING x AS x, y AS y
+ COLUMNS (
+ y text FORMAT JSON PATH '$',
+ NESTED PATH 'strict $[*] ? (@ >= $y)'
+ COLUMNS (
+ z int PATH '$'
+ )
+ )
+ ) jt;
+
+-- Should fail (JSON arguments are not passed to column paths)
+SELECT *
+FROM JSON_TABLE(
+ jsonb '[1,2,3]',
+ '$[*] ? (@ < $x)'
+ PASSING 10 AS x
+ COLUMNS (y text FORMAT JSON PATH '$ ? (@ < $x)')
+ ) jt;
+
+-- Extension: non-constant JSON path
+SELECT JSON_EXISTS(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_VALUE(jsonb '{"a": 123}', '$' || '.' || 'b' DEFAULT 'foo' ON EMPTY);
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a');
+SELECT JSON_QUERY(jsonb '{"a": 123}', '$' || '.' || 'a' WITH WRAPPER);
+-- Should fail (invalid path)
+SELECT JSON_QUERY(jsonb '{"a": 123}', 'error' || ' ' || 'error');
+-- Should fail (not supported)
+SELECT * FROM JSON_TABLE(jsonb '{"a": 123}', '$' || '.' || 'a' COLUMNS (foo int));
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 5ce706205c..e4e525f940 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1311,6 +1311,7 @@ JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
+JsonPathSpec
JsonPathString
JsonPathVarCallback
JsonPathVariable
@@ -1320,6 +1321,17 @@ JsonReturning
JsonScalarExpr
JsonSemAction
JsonSerializeExpr
+JsonTable
+JsonTableColumn
+JsonTableColumnType
+JsonTableContext
+JsonTableJoinState
+JsonTableParent
+JsonTablePlan
+JsonTablePlanJoinType
+JsonTablePlanType
+JsonTableScanState
+JsonTableSibling
JsonTokenType
JsonTransformStringValuesAction
JsonTypeCategory
@@ -2783,6 +2795,7 @@ TableFunc
TableFuncRoutine
TableFuncScan
TableFuncScanState
+TableFuncType
TableInfo
TableLikeClause
TableSampleClause
--
2.35.3
[application/octet-stream] v25-0003-SQL-JSON-query-functions.patch (199.6K, ../../CA+HiwqF14cAgG7e0SJ90VXd8OKPzeRLBAn3p=-R8JV7H+M35fA@mail.gmail.com/6-v25-0003-SQL-JSON-query-functions.patch)
download | inline diff:
From f787434e074296ca57913969fc49bb5cebbacbdf Mon Sep 17 00:00:00 2001
From: Amit Langote <[email protected]>
Date: Thu, 21 Sep 2023 10:14:12 +0900
Subject: [PATCH v25 3/5] SQL/JSON query functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This introduces the SQL/JSON functions for querying JSON data using
jsonpath expressions. The functions are:
JSON_EXISTS()
JSON_QUERY()
JSON_VALUE()
JSON_EXISTS() tests if the jsonpath expression applied to the jsonb
value yields any values.
JSON_VALUE() must return a single value, and an error occurs if it
tries to return multiple values.
JSON_QUERY() must return a json object or array, and there are
various WRAPPER options for handling scalar or multi-value results.
Both these functions have options for handling EMPTY and ERROR
conditions.
All of these functions only operate on jsonb. The workaround for now
is to cast the argument to jsonb.
Author: Nikita Glukhov <[email protected]>
Author: Teodor Sigaev <[email protected]>
Author: Oleg Bartunov <[email protected]>
Author: Alexander Korotkov <[email protected]>
Author: Andrew Dunstan <[email protected]>
Author: Amit Langote <[email protected]>
Reviewers have included (in no particular order) Andres Freund, Alexander
Korotkov, Pavel Stehule, Andrew Alsup, Erik Rijkers, Zihong Yu,
Himanshu Upadhyaya, Daniel Gustafsson, Justin Pryzby, Álvaro Herrera,
jian he, Anton A. Melnikov, Nikita Malakhov
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/[email protected]
Discussion: https://postgr.es/m/abd9b83b-aa66-f230-3d6d-734817f0995d%40postgresql.org
Discussion: https://postgr.es/m/CA+HiwqE4XTdfb1nW=Ojoy_tQSRhYt-q_kb6i5d4xcKyrLC1Nbg@mail.gmail.com
---
doc/src/sgml/func.sgml | 152 +++
src/backend/executor/execExpr.c | 500 +++++++++
src/backend/executor/execExprInterp.c | 405 +++++++
src/backend/jit/llvm/llvmjit.c | 2 +
src/backend/jit/llvm/llvmjit_expr.c | 124 +++
src/backend/jit/llvm/llvmjit_types.c | 4 +
src/backend/nodes/makefuncs.c | 33 +
src/backend/nodes/nodeFuncs.c | 179 +++
src/backend/optimizer/path/costsize.c | 3 +-
src/backend/optimizer/util/clauses.c | 19 +
src/backend/parser/gram.y | 217 +++-
src/backend/parser/parse_expr.c | 553 +++++++++-
src/backend/parser/parse_target.c | 15 +
src/backend/utils/adt/formatting.c | 44 +
src/backend/utils/adt/jsonb.c | 62 ++
src/backend/utils/adt/jsonfuncs.c | 52 +-
src/backend/utils/adt/jsonpath.c | 255 +++++
src/backend/utils/adt/jsonpath_exec.c | 391 ++++++-
src/backend/utils/adt/ruleutils.c | 137 +++
src/include/executor/execExpr.h | 113 ++
src/include/fmgr.h | 1 +
src/include/jit/llvmjit.h | 1 +
src/include/nodes/makefuncs.h | 4 +
src/include/nodes/parsenodes.h | 70 ++
src/include/nodes/primnodes.h | 125 +++
src/include/parser/kwlist.h | 11 +
src/include/utils/formatting.h | 1 +
src/include/utils/jsonb.h | 3 +
src/include/utils/jsonfuncs.h | 5 +
src/include/utils/jsonpath.h | 27 +
src/interfaces/ecpg/preproc/ecpg.trailer | 28 +
src/test/regress/expected/json_sqljson.out | 18 +
src/test/regress/expected/jsonb_sqljson.out | 1097 +++++++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/sql/json_sqljson.sql | 11 +
src/test/regress/sql/jsonb_sqljson.sql | 349 ++++++
src/tools/pgindent/typedefs.list | 20 +
37 files changed, 4966 insertions(+), 67 deletions(-)
create mode 100644 src/test/regress/expected/json_sqljson.out
create mode 100644 src/test/regress/expected/jsonb_sqljson.out
create mode 100644 src/test/regress/sql/json_sqljson.sql
create mode 100644 src/test/regress/sql/jsonb_sqljson.sql
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 202e64d0e0..c12ee06713 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -18170,6 +18170,158 @@ $.* ? (@ like_regex "^\\d+$")
</programlisting>
</para>
</sect3>
+
+ <sect3 id="sqljson-query-functions">
+ <title>SQL/JSON Query Functions</title>
+ <para>
+ <xref linkend="functions-sqljson-querying"/> details the SQL/JSON
+ functions that can be used to query JSON data.
+ </para>
+
+ <note>
+ <para>
+ SQL/JSON path expression can currently only accept values of the
+ <type>jsonb</type> type, so it might be necessary to cast the
+ <replaceable>context_item</replaceable> argument of these functions to
+ <type>jsonb</type>.
+ </para>
+ </note>
+
+ <table id="functions-sqljson-querying">
+ <title>SQL/JSON Query Functions</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ Function signature
+ </para>
+ <para>
+ Description
+ </para>
+ <para>
+ Example(s)
+ </para></entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_exists</primary></indexterm>
+ <function>json_exists</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> </optional>
+ <optional> { <literal>TRUE</literal> | <literal>FALSE</literal> |<literal> UNKNOWN</literal> | <literal>ERROR</literal> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns true if the SQL/JSON <replaceable>path_expression</replaceable>
+ applied to the <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s yields any items.
+ The <literal>ON ERROR</literal> clause specifies what is returned if
+ an error occurs; the default is to return <literal>FALSE</literal>.
+ Note that if the <replaceable>path_expression</replaceable>
+ is <literal>strict</literal>, an error is generated if it yields no items.
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"key1": [1,2,3]}', 'strict $.key1[*] ? (@ > 2)')</literal>
+ <returnvalue>t</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'lax $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>f</returnvalue>
+ </para>
+ <para>
+ <literal>json_exists(jsonb '{"a": [1,2,3]}', 'strict $.a[5]' ERROR ON ERROR)</literal>
+ <returnvalue>ERROR: jsonpath array subscript is out of bounds</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_query</primary></indexterm>
+ <function>json_query</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable> <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> <optional> <literal>FORMAT JSON</literal> <optional> <literal>ENCODING UTF8</literal> </optional> </optional> </optional>
+ <optional> { <literal>WITHOUT</literal> | <literal>WITH</literal> { <literal>CONDITIONAL</literal> | <optional><literal>UNCONDITIONAL</literal></optional> } } <optional> <literal>ARRAY</literal> </optional> <literal>WRAPPER</literal> </optional>
+ <optional> { <literal>KEEP</literal> | <literal>OMIT</literal> } <literal>QUOTES</literal> <optional> <literal>ON SCALAR STRING</literal> </optional> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>EMPTY</literal> { <optional> <literal>ARRAY</literal> </optional> | <literal>OBJECT</literal> } | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <replaceable>value</replaceable>s.
+ This function must return a JSON string, so if the path expression
+ returns multiple SQL/JSON items, you must wrap the result using the
+ <literal>WITH WRAPPER</literal> clause. If the wrapper is
+ <literal>UNCONDITIONAL</literal>, an array wrapper will always
+ be applied, even if the returned value is already a single JSON object
+ or an array, but if it is <literal>CONDITIONAL</literal>, it will not be
+ applied to a single array or object. <literal>UNCONDITIONAL</literal>
+ is the default. If the result is a scalar string, by default the value
+ returned will have surrounding quotes making it a valid JSON value,
+ which can be made explicit by specifying <literal>KEEP QUOTES</literal>.
+ Conversely, quotes can be omitted by specifying <literal>OMIT QUOTES</literal>.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>;
+ the default returned type is <type>jsonb</type>.
+ The <literal>ON EMPTY</literal> clause specifies the behavior if the
+ <replaceable>path_expression</replaceable> yields no value at all; the
+ default when <literal>ON EMPTY</literal> is not specified is to return a
+ null value. The <literal>ON ERROR</literal> clause specifies the behavior
+ if an error occurs as a result of <type>jsonpath</type> evaluation
+ (including cast to the output type) or during the execution of
+ <literal>ON EMPTY</literal> behavior (that was caused by empty result of
+ <type>jsonpath</type> evaluation); the default when
+ <literal>ON ERROR</literal> is not specified is to return a null value.
+ </para>
+ <para>
+ <literal>json_query(jsonb '[1,[2,3],null]', 'lax $[*][1]' WITH CONDITIONAL WRAPPER)</literal>
+ <returnvalue>[3]</returnvalue>
+ </para></entry>
+ </row>
+ <row>
+ <entry role="func_table_entry"><para role="func_signature">
+ <indexterm><primary>json_value</primary></indexterm>
+ <function>json_value</function> (
+ <replaceable>context_item</replaceable>, <replaceable>path_expression</replaceable>
+ <optional> <literal>PASSING</literal> { <replaceable>value</replaceable> <literal>AS</literal> <replaceable>varname</replaceable> } <optional>, ...</optional></optional>
+ <optional> <literal>RETURNING</literal> <replaceable>data_type</replaceable> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON EMPTY</literal> </optional>
+ <optional> { <literal>ERROR</literal> | <literal>NULL</literal> | <literal>DEFAULT</literal> <replaceable>expression</replaceable> } <literal>ON ERROR</literal> </optional>)
+ </para>
+ <para>
+ Returns the result of applying the
+ <replaceable>path_expression</replaceable> to the
+ <replaceable>context_item</replaceable> using the
+ <literal>PASSING</literal> <replaceable>value</replaceable>s. The
+ extracted value must be a single <acronym>SQL/JSON</acronym> scalar
+ item. For results that are objects or arrays, use the
+ <function>json_query</function> function instead.
+ The returned <replaceable>data_type</replaceable> has the same semantics
+ as for constructor functions like <function>json_objectagg</function>.
+ The default returned type is <type>text</type>.
+ The <literal>ON ERROR</literal> and <literal>ON EMPTY</literal>
+ clauses have similar semantics as mentioned in the description of
+ <function>json_query</function>.
+ </para>
+ <para>
+ <literal>json_value(jsonb '"123.45"', '$' RETURNING float)</literal>
+ <returnvalue>123.45</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '"03:04 2015-02-01"', '$.datetime("HH24:MI YYYY-MM-DD")' RETURNING date)</literal>
+ <returnvalue>2015-02-01</returnvalue>
+ </para>
+ <para>
+ <literal>json_value(jsonb '[1,2]', 'strict $[*]' DEFAULT 9 ON ERROR)</literal>
+ <returnvalue>9</returnvalue>
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect3>
</sect2>
</sect1>
diff --git a/src/backend/executor/execExpr.c b/src/backend/executor/execExpr.c
index 34bd2102b5..cfbd92720d 100644
--- a/src/backend/executor/execExpr.c
+++ b/src/backend/executor/execExpr.c
@@ -49,6 +49,7 @@
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/typcache.h"
@@ -88,6 +89,15 @@ static void ExecBuildAggTransCall(ExprState *state, AggState *aggstate,
FunctionCallInfo fcinfo, AggStatePerTrans pertrans,
int transno, int setno, int setoff, bool ishash,
bool nullcheck);
+static void ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch);
+static Datum GetJsonBehaviorConstVal(JsonBehavior *behavior, bool *is_null);
+static int ExecInitJsonCoercion(ExprEvalStep *scratch, ExprState *state,
+ JsonExprState *jsestate,
+ JsonCoercion *coercion,
+ bool throw_errors,
+ Datum *resv, bool *resnull);
/*
@@ -2416,6 +2426,14 @@ ExecInitExprRec(Expr *node, ExprState *state,
break;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+
+ ExecInitJsonExpr(jexpr, state, resv, resnull, &scratch);
+ break;
+ }
+
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -4184,3 +4202,485 @@ ExecBuildParamSetEqual(TupleDesc desc,
return state;
}
+
+/*
+ * Push steps to evaluate a JsonExpr and its various subsidiary expressions.
+ */
+static void
+ExecInitJsonExpr(JsonExpr *jexpr, ExprState *state,
+ Datum *resv, bool *resnull,
+ ExprEvalStep *scratch)
+{
+ JsonExprState *jsestate = palloc0(sizeof(JsonExprState));
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ ListCell *argexprlc;
+ ListCell *argnamelc;
+ int coercion_step_off = -1;
+ int on_error_step_off = -1;
+ List *jumps_if_skip = NIL;
+ List *jumps_to_coerce_finish = NIL;
+ List *jumps_to_coerce_or_end = NIL;
+ List *jumps_to_end = NIL;
+ ListCell *lc;
+ ExprEvalStep *as;
+
+ jsestate->jsexpr = jexpr;
+
+ /*
+ * Evaluate formatted_expr, storing the result into
+ * jsestate->formatted_expr.
+ */
+ ExecInitExprRec((Expr *) jexpr->formatted_expr, state,
+ &jsestate->formatted_expr.value,
+ &jsestate->formatted_expr.isnull);
+
+ /*
+ * Steps to JUMP to end if formatted_expr evaluated to NULL, skipping over
+ * next steps including the JsonPath evaluation.
+ */
+ jumps_if_skip = lappend_int(jumps_if_skip, state->steps_len);
+ scratch->opcode = EEOP_JUMP_IF_NULL;
+ scratch->resnull = &jsestate->formatted_expr.isnull;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Evaluate pathspec expression, storing the result into
+ * jsestate->pathspec.
+ */
+ ExecInitExprRec((Expr *) jexpr->path_spec, state,
+ &jsestate->pathspec.value,
+ &jsestate->pathspec.isnull);
+
+ /*
+ * Steps to JUMP to end if pathspec evaluated to NULL, skipping over
+ * next steps including the JsonPath evaluation.
+ */
+ jumps_if_skip = lappend_int(jumps_if_skip, state->steps_len);
+ scratch->opcode = EEOP_JUMP_IF_NULL;
+ scratch->resnull = &jsestate->pathspec.isnull;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+
+ /* Steps to compute PASSING args. */
+ jsestate->args = NIL;
+ forboth(argexprlc, jexpr->passing_values,
+ argnamelc, jexpr->passing_names)
+ {
+ Expr *argexpr = (Expr *) lfirst(argexprlc);
+ String *argname = lfirst_node(String, argnamelc);
+ JsonPathVariable *var = palloc(sizeof(*var));
+
+ var->name = argname->sval;
+ var->typid = exprType((Node *) argexpr);
+ var->typmod = exprTypmod((Node *) argexpr);
+
+ ExecInitExprRec((Expr *) argexpr, state, &var->value, &var->isnull);
+
+ jsestate->args = lappend(jsestate->args, var);
+ }
+
+ /*
+ * Step for the actual JsonPath* evaluation; see ExecEvalJsonExprPath().
+ */
+ scratch->opcode = EEOP_JSONEXPR_PATH;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Step to jump to end when there's neither an error nor any need to coerce
+ * the JsonPath* result.
+ */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Steps to coerce the JsonPath* result computed by
+ * ExecEvalJsonExprPath(). To handle coercion errors softly, use the
+ * following ErrorSaveContext when initializing the coercion expressions
+ * and in ExecEvalJsonCoercionViaPopulateOrIO().
+ */
+ jsestate->escontext.type = T_ErrorSaveContext;
+ if (jexpr->result_coercion || jexpr->omit_quotes)
+ {
+ coercion_step_off = state->steps_len;
+
+ jsestate->jump_eval_result_coercion =
+ ExecInitJsonCoercion(scratch, state, jsestate,
+ jexpr->result_coercion,
+ jexpr->on_error->btype == JSON_BEHAVIOR_ERROR,
+ resv, resnull);
+
+ /*
+ * Step to jump to the EEOP_JSONEXPR_FINISH step skipping over item
+ * coercion steps that will be added below, if any.
+ */
+ if (jexpr->item_coercions)
+ {
+ jumps_to_coerce_finish = lappend_int(jumps_to_coerce_finish,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+ else
+ jsestate->jump_eval_result_coercion = -1;
+
+ /* Steps for coercions of JsonItemType values for JSON_VALUE_OP. */
+ if (jexpr->item_coercions)
+ {
+ if (coercion_step_off < 0)
+ coercion_step_off = state->steps_len;
+
+ /*
+ * ExecPrepareJsonItemCoercion() called by ExecEvalJsonExprPath()
+ * chooses one from the array for a given JsonbValue returned by
+ * JsonPathValue(), indexed by JsonItemType of a given
+ * JsonbValue.
+ */
+ jsestate->num_item_coercions = list_length(jexpr->item_coercions);
+ jsestate->eval_item_coercion_jumps = (int *)
+ palloc(jsestate->num_item_coercions * sizeof(int));
+ jsestate->item_coercion_via_expr = (bool *)
+ palloc(jsestate->num_item_coercions * sizeof(bool));
+ foreach(lc, jexpr->item_coercions)
+ {
+ JsonItemCoercion *item_coercion = lfirst(lc);
+ JsonCoercion *coercion = item_coercion->coercion;
+
+ jsestate->item_coercion_via_expr[item_coercion->item_type] =
+ item_coercion->coercion->expr != NULL;
+ jsestate->eval_item_coercion_jumps[item_coercion->item_type] =
+ ExecInitJsonCoercion(scratch, state, jsestate,
+ coercion,
+ jexpr->on_error->btype == JSON_BEHAVIOR_ERROR,
+ resv, resnull);
+
+ /* Emit JUMP step to skip past other coercions' steps. */
+ jumps_to_coerce_finish = lappend_int(jumps_to_coerce_finish,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /*
+ * And a step to clean up after the coercion step; see
+ * ExecEvalJsonCoercionFinish(). Its main role is to check if an
+ * error occurred when evaluating the coercion and handle it per the
+ * specified ON ERROR behavior.
+ */
+ if (coercion_step_off >= 0)
+ {
+ foreach(lc, jumps_to_coerce_finish)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+
+ scratch->opcode = EEOP_JSONEXPR_COERCION_FINISH;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+ }
+
+ /*
+ * Step to handle non-ERROR ON ERROR behaviors. That also handles errors
+ * that may occur during coercion handling.
+ */
+ if (jexpr->on_error &&
+ jexpr->on_error->btype != JSON_BEHAVIOR_ERROR)
+ {
+ on_error_step_off = state->steps_len;
+ if (jexpr->on_empty == NULL ||
+ jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
+ jumps_to_end = lappend_int(jumps_to_end, on_error_step_off);
+ scratch->opcode = EEOP_JUMP_IF_NOT_TRUE;
+ /*
+ * post_eval.error is set as appropriate by ExecEvalJsonExprPath() and
+ * ExecEvalJsonCoercionFinish() to trigger the ON ERROR step.
+ */
+ scratch->resvalue = &post_eval->error.value;
+ scratch->resnull = &post_eval->error.isnull;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /* Step(s) to evaluate the ON ERROR expression */
+ if (jexpr->on_error->default_expr)
+ {
+ ErrorSaveContext *save_escontext = state->escontext;
+
+ state->escontext = &jsestate->escontext;
+ ExecInitExprRec((Expr *) jexpr->on_error->default_expr,
+ state, resv, resnull);
+ state->escontext = save_escontext;
+
+ /* Jump to end, because no coercion needed. */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* set below */
+ ExprEvalPushStep(state, scratch);
+ }
+ else
+ {
+ Datum constvalue;
+ bool constisnull;
+
+ constvalue = GetJsonBehaviorConstVal(jexpr->on_error,
+ &constisnull);
+ scratch->opcode = EEOP_CONST;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.constval.value = constvalue;
+ scratch->d.constval.isnull = constisnull;
+
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Jump to the coercion step to coerce the above value to the
+ * desired output type.
+ */
+ jumps_to_coerce_or_end = lappend_int(jumps_to_coerce_or_end,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /* Step to handle non-ERROR ON EMPTY behaviors. */
+ if (jexpr->on_empty != NULL &&
+ jexpr->on_empty->btype != JSON_BEHAVIOR_ERROR)
+ {
+ /* Jump to handle ON EMPTY after checking error. */
+ if (on_error_step_off >= 0)
+ {
+ as = &state->steps[on_error_step_off];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+ /*
+ * If no step was added above, jump to handle ON EMPTY directly
+ * instead.
+ */
+ else
+ on_error_step_off = state->steps_len;
+
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP_IF_NOT_TRUE;
+ scratch->resvalue = &post_eval->empty.value;
+ scratch->resnull = &post_eval->empty.isnull;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /* Step(s) to evaluate the ON EMPTY expression */
+ if (jexpr->on_empty->default_expr)
+ {
+ ErrorSaveContext *save_escontext = state->escontext;
+
+ state->escontext = &jsestate->escontext;
+ ExecInitExprRec((Expr *) jexpr->on_empty->default_expr,
+ state, resv, resnull);
+ state->escontext = save_escontext;
+
+ /*
+ * Emit JUMP step to jump to the end to skip over the CONST step
+ * that will be added below. Note no coercion needed.
+ */
+ jumps_to_end = lappend_int(jumps_to_end, state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+ }
+ else
+ {
+ Datum constvalue;
+ bool constisnull;
+
+ constvalue = GetJsonBehaviorConstVal(jexpr->on_empty,
+ &constisnull);
+ scratch->opcode = EEOP_CONST;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.constval.value = constvalue;
+ scratch->d.constval.isnull = constisnull;
+
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Emit JUMP step to jump to the coercion step to coerce the above
+ * value to the desired output type.
+ */
+ jumps_to_coerce_or_end = lappend_int(jumps_to_coerce_or_end,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1;
+ ExprEvalPushStep(state, scratch);
+ }
+ }
+
+ /*
+ * Return NULL on skipping JsonPath* evaluation when either formatted_expr
+ * or pathspec is NULL.
+ */
+ foreach(lc, jumps_if_skip)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+ scratch->opcode = EEOP_CONST;
+ scratch->resvalue = resv;
+ scratch->resnull = resnull;
+ scratch->d.constval.value = (Datum) 0;
+ scratch->d.constval.isnull = true;
+ ExprEvalPushStep(state, scratch);
+
+ /* Do put that NULL through coercion though. */
+ jumps_to_coerce_or_end = lappend_int(jumps_to_coerce_or_end,
+ state->steps_len);
+ scratch->opcode = EEOP_JUMP;
+ scratch->d.jump.jumpdone = -1; /* computed later */
+ ExprEvalPushStep(state, scratch);
+
+ /*
+ * Adjust remaining jump target addresses now that we have the necessary
+ * steps in place.
+ */
+ Assert(on_error_step_off >= 0 || jexpr == NULL ||
+ jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
+ jsestate->jump_error = on_error_step_off;
+
+ /* Adjust EEOP_JUMP steps */
+ foreach(lc, jumps_to_coerce_or_end)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = coercion_step_off >= 0 ?
+ coercion_step_off : state->steps_len;
+ }
+
+ foreach(lc, jumps_to_end)
+ {
+ as = &state->steps[lfirst_int(lc)];
+ as->d.jump.jumpdone = state->steps_len;
+ }
+
+ /*
+ * Set information for RETURNING type's input function used by
+ * ExecEvalJsonExprCoercion().
+ */
+ if (jexpr->omit_quotes ||
+ (jexpr->result_coercion && jexpr->result_coercion->via_io))
+ {
+ Oid typinput;
+ FmgrInfo *finfo;
+
+ /* lookup the result type's input function */
+ getTypeInputInfo(jexpr->returning->typid, &typinput,
+ &jsestate->input.typioparam);
+ finfo = palloc0(sizeof(FmgrInfo));
+ fmgr_info(typinput, finfo);
+ jsestate->input.finfo = finfo;
+ }
+}
+
+/*
+ * Returns constant values to be returned to the user for various
+ * non-ERROR ON ERROR/EMPTY behaviors.
+ *
+ * Note that JSON_BEHAVIOR_DEFAULT expression is handled by the
+ * caller separately.
+ */
+static Datum
+GetJsonBehaviorConstVal(JsonBehavior *behavior, bool *is_null)
+{
+ *is_null = false;
+
+ switch (behavior->btype)
+ {
+ case JSON_BEHAVIOR_EMPTY_ARRAY:
+ return JsonbPGetDatum(JsonbMakeEmptyArray());
+
+ case JSON_BEHAVIOR_EMPTY_OBJECT:
+ return JsonbPGetDatum(JsonbMakeEmptyObject());
+
+ case JSON_BEHAVIOR_TRUE:
+ return BoolGetDatum(true);
+
+ case JSON_BEHAVIOR_FALSE:
+ return BoolGetDatum(false);
+
+ case JSON_BEHAVIOR_NULL:
+ case JSON_BEHAVIOR_UNKNOWN:
+ case JSON_BEHAVIOR_EMPTY:
+ *is_null = true;
+ return (Datum) 0;
+
+ case JSON_BEHAVIOR_DEFAULT:
+ /* Always handled in the caller. */
+ Assert(false);
+ return (Datum) 0;
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON behavior %d", behavior->btype);
+ return (Datum) 0;
+ }
+}
+
+/* Initialize one JsonCoercion for execution. */
+static int
+ExecInitJsonCoercion(ExprEvalStep *scratch, ExprState *state,
+ JsonExprState *jsestate,
+ JsonCoercion *coercion,
+ bool throw_errors,
+ Datum *resv, bool *resnull)
+{
+ int jump_eval_coercion;
+
+ if (jsestate->jsexpr->omit_quotes ||
+ (coercion && (coercion->via_io || coercion->via_populate)))
+ {
+ jump_eval_coercion = state->steps_len;
+ scratch->opcode = EEOP_JSONEXPR_COERCION;
+ scratch->d.jsonexpr.jsestate = jsestate;
+ ExprEvalPushStep(state, scratch);
+ }
+ else if (coercion && coercion->expr)
+ {
+ Datum *save_innermost_caseval;
+ bool *save_innermost_casenull;
+ ErrorSaveContext *save_escontext;
+
+ /* Push step(s) to compute cstate->coercion. */
+ save_innermost_caseval = state->innermost_caseval;
+ save_innermost_casenull = state->innermost_casenull;
+ save_escontext = state->escontext;
+
+ state->innermost_caseval = resv;
+ state->innermost_casenull = resnull;
+ if (!throw_errors)
+ state->escontext = &jsestate->escontext;
+ else
+ state->escontext = NULL;
+
+ jump_eval_coercion = state->steps_len;
+ ExecInitExprRec((Expr *) coercion->expr, state, resv, resnull);
+
+ state->innermost_caseval = save_innermost_caseval;
+ state->innermost_casenull = save_innermost_casenull;
+ state->escontext = save_escontext;
+ }
+ else
+ {
+ /*
+ * Coercion is unnecessary; for example, RETURNING type matches JSON
+ * item's type exactly.
+ */
+ jump_eval_coercion = -1;
+ }
+
+ return jump_eval_coercion;
+}
diff --git a/src/backend/executor/execExprInterp.c b/src/backend/executor/execExprInterp.c
index 4e152fdfe3..74217fb59d 100644
--- a/src/backend/executor/execExprInterp.c
+++ b/src/backend/executor/execExprInterp.c
@@ -75,6 +75,7 @@
#include "utils/json.h"
#include "utils/jsonb.h"
#include "utils/jsonfuncs.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/timestamp.h"
@@ -153,6 +154,10 @@ static TupleDesc get_cached_rowtype(Oid type_id, int32 typmod,
bool *changed);
static void ExecEvalRowNullInt(ExprState *state, ExprEvalStep *op,
ExprContext *econtext, bool checkisnull);
+static Datum ExecPrepareJsonItemCoercion(JsonbValue *item,
+ JsonExprState *jsestate,
+ int *jumps_eval_item_coercion,
+ bool *via_expr);
/* fast-path evaluation functions */
static Datum ExecJustInnerVar(ExprState *state, ExprContext *econtext, bool *isnull);
@@ -482,6 +487,9 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
&&CASE_EEOP_XMLEXPR,
&&CASE_EEOP_JSON_CONSTRUCTOR,
&&CASE_EEOP_IS_JSON,
+ &&CASE_EEOP_JSONEXPR_PATH,
+ &&CASE_EEOP_JSONEXPR_COERCION,
+ &&CASE_EEOP_JSONEXPR_COERCION_FINISH,
&&CASE_EEOP_AGGREF,
&&CASE_EEOP_GROUPING_FUNC,
&&CASE_EEOP_WINDOW_FUNC,
@@ -1551,6 +1559,35 @@ ExecInterpExpr(ExprState *state, ExprContext *econtext, bool *isnull)
EEO_NEXT();
}
+ EEO_CASE(EEOP_JSONEXPR_PATH)
+ {
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+
+ /* too complex for an inline implementation */
+ if (!ExecEvalJsonExprPath(state, op, econtext))
+ EEO_JUMP(jsestate->jump_error);
+ else if (jsestate->post_eval.jump_eval_coercion >= 0)
+ EEO_JUMP(jsestate->post_eval.jump_eval_coercion);
+
+ EEO_NEXT();
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION)
+ {
+ /* too complex for an inline implementation */
+ ExecEvalJsonCoercionViaPopulateOrIO(state, op, econtext);
+
+ EEO_NEXT();
+ }
+
+ EEO_CASE(EEOP_JSONEXPR_COERCION_FINISH)
+ {
+ /* too complex for an inline implementation */
+ ExecEvalJsonCoercionFinish(state, op);
+
+ EEO_NEXT();
+ }
+
EEO_CASE(EEOP_AGGREF)
{
/*
@@ -4208,6 +4245,374 @@ ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op)
*op->resvalue = BoolGetDatum(res);
}
+/*
+ * Performs JsonPath{Exists|Query|Value}() for given context item and JSON
+ * path.
+ *
+ * Result is set in *op->resvalue and *op->resnull.
+ *
+ * On return, JsonExprPostEvalState is populated with the following details:
+ * - jump_eval_coercion: step address of coercion to apply to the result
+ * - error.value: true if an error occurred during JsonPath evaluation
+ * - empty.value: true if JsonPath{Query|Value}() found no matching item
+ *
+ * No return if the ON ERROR/EMPTY behavior is ERROR.
+ */
+bool
+ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonExpr *jexpr = jsestate->jsexpr;
+ Datum item;
+ JsonPath *path;
+ bool throw_error = (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR);
+ bool error = false,
+ empty = false;
+
+ item = jsestate->formatted_expr.value;
+ path = DatumGetJsonPathP(jsestate->pathspec.value);
+
+ /* Reset JsonExprPostEvalState for this evaluation. */
+ memset(post_eval, 0, sizeof(*post_eval));
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ {
+ bool exists = JsonPathExists(item, path,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (!error)
+ {
+ *op->resvalue = BoolGetDatum(exists);
+ *op->resnull = false;
+ }
+ }
+ break;
+
+ case JSON_QUERY_OP:
+ *op->resvalue = JsonPathQuery(item, path, jexpr->wrapper, &empty,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (!error && !empty)
+ *op->resnull = (DatumGetPointer(*op->resvalue) == NULL);
+ break;
+
+ case JSON_VALUE_OP:
+ {
+ JsonbValue *jbv = JsonPathValue(item, path, &empty,
+ !throw_error ? &error : NULL,
+ jsestate->args);
+
+ /* Might get overridden by an item coercion below. */
+ post_eval->jump_eval_coercion = jsestate->jump_eval_result_coercion;
+ if (jbv == NULL)
+ {
+ /* Will be coerced with result_coercion. */
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ }
+ else if (!error && !empty)
+ {
+ Assert(jbv != NULL);
+
+ /*
+ * If the requested output type is json(b), use
+ * result_coercion to do the coercion.
+ */
+ if (jexpr->returning->typid == JSONOID ||
+ jexpr->returning->typid == JSONBOID)
+ {
+ *op->resvalue = JsonbPGetDatum(JsonbValueToJsonb(jbv));
+ *op->resnull = false;
+ }
+ else
+ {
+ bool via_expr;
+ int jump_eval_item_coercion;
+
+ /*
+ * Else, use one of the item_coercions.
+ *
+ * Error out if no cast expression exists.
+ */
+ *op->resvalue = ExecPrepareJsonItemCoercion(jbv, jsestate,
+ &jump_eval_item_coercion,
+ &via_expr);
+ *op->resnull = false;
+ if (jump_eval_item_coercion >= 0 && !via_expr)
+ {
+ if (!throw_error)
+ {
+ /* Will be coerced with result_coercion. */
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ }
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_ITEM_CANNOT_BE_CAST_TO_TARGET_TYPE),
+ errmsg("SQL/JSON item cannot be cast to target type")));
+ }
+ post_eval->jump_eval_coercion = jump_eval_item_coercion;
+ }
+ }
+ break;
+ }
+
+ default:
+ elog(ERROR, "unrecognized SQL/JSON expression op %d", jexpr->op);
+ return false;
+ }
+
+ if (error)
+ {
+ Assert(!throw_error && jsestate->jump_error >= 0);
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ post_eval->error.value = BoolGetDatum(true);
+ return false;
+ }
+
+ if (empty)
+ {
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ if (jexpr->on_empty &&
+ jexpr->on_empty->btype == JSON_BEHAVIOR_ERROR)
+ {
+ if (jexpr->on_error->btype == JSON_BEHAVIOR_ERROR)
+ ereport(ERROR,
+ (errcode(ERRCODE_NO_SQL_JSON_ITEM),
+ errmsg("no SQL/JSON item")));
+ post_eval->error.value = BoolGetDatum(true);
+ }
+
+ post_eval->empty.value = BoolGetDatum(true);
+ return false;
+ }
+
+ post_eval->throw_coercion_error = throw_error;
+ return true;
+}
+
+
+/*
+ * Evaluate or return the step address to evaluate a coercion of a JSON item
+ * to the target type. The former if the coercion must be done right away by
+ * calling the target type's input function, and for some types, by calling
+ * json_populate_type().
+ *
+ * Returns the step address to be performed next.
+ */
+void
+ExecEvalJsonCoercionViaPopulateOrIO(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ JsonExpr *jexpr = jsestate->jsexpr;
+ Node *escontext_p = jexpr->on_error->btype != JSON_BEHAVIOR_ERROR ?
+ (Node *) &jsestate->escontext : NULL;
+ JsonCoercion *coercion = jexpr->result_coercion;
+ Datum res = *op->resvalue;
+ bool resnull = *op->resnull;
+ bool type_is_domain =
+ (getBaseType(jexpr->returning->typid) != jexpr->returning->typid);
+
+ Assert(coercion != NULL || jexpr->omit_quotes);
+
+ /*
+ * Force-throw an error if the returning type is a domain because its
+ * constraint violations must be reported, or if coercing a behavior
+ * value.
+ */
+ if (type_is_domain || post_eval->throw_coercion_error)
+ escontext_p = NULL;
+
+ if (coercion && coercion->via_populate)
+ {
+ *op->resvalue = json_populate_type(res, JSONBOID,
+ jexpr->returning->typid,
+ jexpr->returning->typmod,
+ &jsestate->cache,
+ econtext->ecxt_per_query_memory,
+ op->resnull,
+ escontext_p);
+ if (SOFT_ERROR_OCCURRED(escontext_p))
+ {
+ post_eval->error.value = BoolGetDatum(true);
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ }
+ }
+ else if ((coercion && coercion->via_io) || jexpr->omit_quotes)
+ {
+ char *val_string = resnull ? NULL :
+ JsonbUnquote(DatumGetJsonbP(res));
+
+ if (!InputFunctionCallSafe(jsestate->input.finfo, val_string,
+ jsestate->input.typioparam,
+ jexpr->returning->typmod,
+ escontext_p,
+ op->resvalue))
+ {
+ post_eval->error.value = BoolGetDatum(true);
+ *op->resnull = true;
+ *op->resvalue = (Datum) 0;
+ }
+ }
+}
+
+/*
+ * Checks if the coercion evaluation (either an expression or
+ * ExecEvalJsonCoercionViaPopulateOrIO()) led to an error. If an error did
+ * occur, this sets post_eval->error to trigger the subsequent ON ERROR
+ * handling steps.
+ */
+void
+ExecEvalJsonCoercionFinish(ExprState *state, ExprEvalStep *op)
+{
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+
+ if (SOFT_ERROR_OCCURRED(&jsestate->escontext))
+ {
+ *op->resvalue = (Datum) 0;
+ *op->resnull = true;
+ jsestate->post_eval.error.value = BoolGetDatum(true);
+
+ /*
+ * Also make ErrorSaveContext ready for the next row. Since we never
+ * set details_wanted, we don't need to also reset error_data, which
+ * would be NULL anyway.
+ */
+ Assert(!jsestate->escontext.details_wanted &&
+ jsestate->escontext.error_data == NULL);
+ jsestate->escontext.error_occurred = false;
+
+ /*
+ * Finally, throw any errors that may occur when coercing the ON ERROR
+ * behavion expression, if any.
+ */
+ jsestate->post_eval.throw_coercion_error = true;
+ }
+ else
+ {
+ /*
+ * If no error, reset error/empty flags so that ON ERROR/EMPTY handling
+ * steps are skipped over.
+ */
+ jsestate->post_eval.error.value = BoolGetDatum(false);
+ jsestate->post_eval.empty.value = BoolGetDatum(false);
+ }
+}
+
+/*
+ * Prepare SQL/JSON item coercion to the output type. Returned a datum of the
+ * corresponding SQL type and a pointer to the coercion state.
+ */
+static Datum
+ExecPrepareJsonItemCoercion(JsonbValue *item, JsonExprState *jsestate,
+ int *jump_eval_item_coercion,
+ bool *via_expr)
+{
+ int *eval_item_coercion_jumps = jsestate->eval_item_coercion_jumps;
+ bool *item_coercion_via_expr = jsestate->item_coercion_via_expr;
+ int jump_to;
+ Datum res;
+ JsonbValue buf;
+
+ if (item->type == jbvBinary &&
+ JsonContainerIsScalar(item->val.binary.data))
+ {
+ bool res PG_USED_FOR_ASSERTS_ONLY;
+
+ res = JsonbExtractScalar(item->val.binary.data, &buf);
+ item = &buf;
+ Assert(res);
+ }
+
+ /* get coercion state reference and datum of the corresponding SQL type */
+ switch (item->type)
+ {
+ case jbvNull:
+ *via_expr = item_coercion_via_expr[JsonItemTypeNull];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeNull];
+ res = (Datum) 0;
+ break;
+
+ case jbvString:
+ *via_expr = item_coercion_via_expr[JsonItemTypeString];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeString];
+ res = PointerGetDatum(cstring_to_text_with_len(item->val.string.val,
+ item->val.string.len));
+ break;
+
+ case jbvNumeric:
+ *via_expr = item_coercion_via_expr[JsonItemTypeNumeric];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeNumeric];
+ res = NumericGetDatum(item->val.numeric);
+ break;
+
+ case jbvBool:
+ *via_expr = item_coercion_via_expr[JsonItemTypeBoolean];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeBoolean];
+ res = BoolGetDatum(item->val.boolean);
+ break;
+
+ case jbvDatetime:
+ res = item->val.datetime.value;
+ switch (item->val.datetime.typid)
+ {
+ case DATEOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeDate];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeDate];
+ break;
+ case TIMEOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTime];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTime];
+ break;
+ case TIMETZOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTimetz];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimetz];
+ break;
+ case TIMESTAMPOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTimestamp];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimestamp];
+ break;
+ case TIMESTAMPTZOID:
+ *via_expr = item_coercion_via_expr[JsonItemTypeTimestamptz];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeTimestamptz];
+ break;
+ default:
+ elog(ERROR, "unexpected jsonb datetime type oid %u",
+ item->val.datetime.typid);
+ return (Datum) 0;
+ }
+ break;
+
+ case jbvArray:
+ case jbvObject:
+ case jbvBinary:
+ *via_expr = item_coercion_via_expr[JsonItemTypeComposite];
+ jump_to = eval_item_coercion_jumps[JsonItemTypeComposite];
+ res = JsonbPGetDatum(JsonbValueToJsonb(item));
+ break;
+
+ default:
+ elog(ERROR, "unexpected jsonb value type %d", item->type);
+ return (Datum) 0;
+ }
+
+ *jump_eval_item_coercion = jump_to;
+
+ return res;
+}
+
/*
* ExecEvalGroupingFunc
diff --git a/src/backend/jit/llvm/llvmjit.c b/src/backend/jit/llvm/llvmjit.c
index 2c8ac02550..4e83e5ef7f 100644
--- a/src/backend/jit/llvm/llvmjit.c
+++ b/src/backend/jit/llvm/llvmjit.c
@@ -84,6 +84,7 @@ LLVMTypeRef StructAggState;
LLVMTypeRef StructAggStatePerGroupData;
LLVMTypeRef StructAggStatePerTransData;
LLVMTypeRef StructPlanState;
+LLVMTypeRef StructJsonExprPostEvalState;
LLVMValueRef AttributeTemplate;
LLVMValueRef ExecEvalSubroutineTemplate;
@@ -1186,6 +1187,7 @@ llvm_create_types(void)
StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
StructPlanState = llvm_pg_var_type("StructPlanState");
StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
+ StructJsonExprPostEvalState = llvm_pg_var_type("StructJsonExprPostEvalState");
AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index 81856a9dc7..901324c988 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -1930,6 +1930,130 @@ llvm_compile_expr(ExprState *state)
LLVMBuildBr(b, opblocks[opno + 1]);
break;
+ case EEOP_JSONEXPR_PATH:
+ {
+ JsonExprState *jsestate = op->d.jsonexpr.jsestate;
+ LLVMValueRef v_ret;
+ LLVMBasicBlockRef b_coercion;
+
+ b_coercion =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_coercion", opno);
+
+ v_ret = build_EvalXFunc(b, mod, "ExecEvalJsonExprPath",
+ v_state, op, v_econtext);
+ v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, "");
+ LLVMBuildCondBr(b,
+ LLVMBuildICmp(b,
+ LLVMIntEQ,
+ v_ret,
+ l_sbool_const(0),
+ ""),
+ jsestate->jump_error >= 0 ?
+ opblocks[jsestate->jump_error] :
+ b_coercion,
+ b_coercion);
+
+ LLVMPositionBuilderAtEnd(b, b_coercion);
+ if (jsestate->jump_eval_result_coercion >= 0 ||
+ jsestate->num_item_coercions > 0)
+ {
+ JsonExprPostEvalState *post_eval = &jsestate->post_eval;
+ int i;
+ LLVMValueRef v_post_eval;
+ LLVMValueRef v_post_eval_jump_eval_coercionp;
+ LLVMValueRef v_post_eval_jump_eval_coercion;
+ LLVMValueRef v_jump_coercion;
+ LLVMValueRef v_switch;
+ LLVMBasicBlockRef b_done,
+ b_result_coercion_block,
+ *b_item_coercion_blocks = NULL;
+
+ v_post_eval = l_ptr_const(post_eval, l_ptr(StructJsonExprPostEvalState));
+ v_post_eval_jump_eval_coercionp =
+ l_struct_gep(b,
+ StructJsonExprPostEvalState,
+ v_post_eval,
+ FIELDNO_JSONEXPRPOSTEVALSTATE_JUMP_EVAL_COERCION,
+ "v_post_eval_jump_eval_coercion");
+ v_post_eval_jump_eval_coercion =
+ l_load(b, LLVMInt32TypeInContext(lc),
+ v_post_eval_jump_eval_coercionp, "");
+
+ b_result_coercion_block =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_result_coercion", opno);
+ if (jsestate->num_item_coercions > 0)
+ {
+ b_item_coercion_blocks = palloc(sizeof(LLVMBasicBlockRef) *
+ jsestate->num_item_coercions);
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ b_item_coercion_blocks[i] =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_item_coercion.%d",
+ opno, i);
+ }
+ }
+ b_done =
+ l_bb_before_v(opblocks[opno + 1],
+ "op.%d.jsonexpr_done", opno);
+
+ v_switch = LLVMBuildSwitch(b,
+ v_post_eval_jump_eval_coercion,
+ b_done,
+ jsestate->num_item_coercions + 1);
+ if (jsestate->jump_eval_result_coercion >= 0)
+ {
+ v_jump_coercion = l_int32_const(lc, jsestate->jump_eval_result_coercion);
+ LLVMAddCase(v_switch, v_jump_coercion, b_result_coercion_block);
+ }
+
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ if (jsestate->eval_item_coercion_jumps[i] >= 0)
+ {
+ v_jump_coercion = l_int32_const(lc, jsestate->eval_item_coercion_jumps[i]);
+ LLVMAddCase(v_switch, v_jump_coercion, b_item_coercion_blocks[i]);
+ }
+ }
+
+ LLVMPositionBuilderAtEnd(b, b_result_coercion_block);
+ if (jsestate->jump_eval_result_coercion >= 0)
+ LLVMBuildBr(b, opblocks[jsestate->jump_eval_result_coercion]);
+ else
+ LLVMBuildUnreachable(b);
+
+ for (i = 0; i < jsestate->num_item_coercions; i++)
+ {
+ LLVMPositionBuilderAtEnd(b, b_item_coercion_blocks[i]);
+ if (jsestate->eval_item_coercion_jumps[i] >= 0)
+ LLVMBuildBr(b, opblocks[jsestate->eval_item_coercion_jumps[i]]);
+ else
+ LLVMBuildUnreachable(b);
+ }
+
+ LLVMPositionBuilderAtEnd(b, b_done);
+ }
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+ }
+
+ case EEOP_JSONEXPR_COERCION:
+ build_EvalXFunc(b, mod, "ExecEvalJsonCoercionViaPopulateOrIO",
+ v_state, op, v_econtext);
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
+ case EEOP_JSONEXPR_COERCION_FINISH:
+ build_EvalXFunc(b, mod, "ExecEvalJsonCoercionFinish",
+ v_state, op);
+
+ LLVMBuildBr(b, opblocks[opno + 1]);
+ break;
+
case EEOP_AGGREF:
{
LLVMValueRef v_aggno;
diff --git a/src/backend/jit/llvm/llvmjit_types.c b/src/backend/jit/llvm/llvmjit_types.c
index 3a4be09e50..ce155119b4 100644
--- a/src/backend/jit/llvm/llvmjit_types.c
+++ b/src/backend/jit/llvm/llvmjit_types.c
@@ -69,6 +69,7 @@ MinimalTupleTableSlot StructMinimalTupleTableSlot;
TupleDescData StructTupleDescData;
PlanState StructPlanState;
MinimalTupleData StructMinimalTupleData;
+JsonExprPostEvalState StructJsonExprPostEvalState;
/*
@@ -172,6 +173,9 @@ void *referenced_functions[] =
ExecEvalXmlExpr,
ExecEvalJsonConstructor,
ExecEvalJsonIsPredicate,
+ ExecEvalJsonCoercionViaPopulateOrIO,
+ ExecEvalJsonCoercionFinish,
+ ExecEvalJsonExprPath,
MakeExpandedObjectReadOnlyInternal,
slot_getmissingattrs,
slot_getsomeattrs_int,
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index c6fb571982..746ed3d1da 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -857,6 +857,39 @@ makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
return jve;
}
+/*
+ * makeJsonBehaviorClause -
+ * creates a JsonBehaviorClause node
+ */
+JsonBehaviorClause *
+makeJsonBehaviorClause(JsonBehavior *on_empty, JsonBehavior *on_error,
+ int location)
+{
+ JsonBehaviorClause *behavior = makeNode(JsonBehaviorClause);
+
+ behavior->on_empty = on_empty;
+ behavior->on_error = on_error;
+ behavior->location = location;
+
+ return behavior;
+}
+
+/*
+ * makeJsonBehavior -
+ * creates a JsonBehavior node
+ */
+JsonBehavior *
+makeJsonBehavior(JsonBehaviorType type, Node *default_expr, int location)
+{
+ JsonBehavior *behavior = makeNode(JsonBehavior);
+
+ behavior->btype = type;
+ behavior->default_expr = default_expr;
+ behavior->location = location;
+
+ return behavior;
+}
+
/*
* makeJsonEncoding -
* converts JSON encoding name to enum JsonEncoding
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index c03f4f23e2..8e49ef6b95 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -234,6 +234,15 @@ exprType(const Node *expr)
case T_JsonIsPredicate:
type = BOOLOID;
break;
+ case T_JsonExpr:
+ type = ((const JsonExpr *) expr)->returning->typid;
+ break;
+ case T_JsonItemCoercion:
+ type = exprType((Node *) ((const JsonItemCoercion *) expr)->coercion);
+ break;
+ case T_JsonCoercion:
+ type = exprType(((const JsonCoercion *) expr)->expr);
+ break;
case T_NullTest:
type = BOOLOID;
break;
@@ -493,6 +502,12 @@ exprTypmod(const Node *expr)
return exprTypmod((Node *) ((const JsonValueExpr *) expr)->formatted_expr);
case T_JsonConstructorExpr:
return ((const JsonConstructorExpr *) expr)->returning->typmod;
+ case T_JsonExpr:
+ return ((JsonExpr *) expr)->returning->typmod;
+ case T_JsonItemCoercion:
+ return exprTypmod((Node *) ((const JsonItemCoercion *) expr)->coercion);
+ case T_JsonCoercion:
+ return exprTypmod(((const JsonCoercion *) expr)->expr);
case T_CoerceToDomain:
return ((const CoerceToDomain *) expr)->resulttypmod;
case T_CoerceToDomainValue:
@@ -969,6 +984,22 @@ exprCollation(const Node *expr)
/* IS JSON's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) expr;
+ JsonCoercion *coercion = jexpr->result_coercion;
+
+ if (!coercion)
+ coll = InvalidOid;
+ else if (coercion->expr)
+ coll = exprCollation(coercion->expr);
+ else if (coercion->via_io || coercion->via_populate)
+ coll = coercion->collation;
+ else
+ coll = InvalidOid;
+ }
+ break;
+
case T_NullTest:
/* NullTest's result is boolean ... */
coll = InvalidOid; /* ... so it has no collation */
@@ -1160,6 +1191,9 @@ exprSetCollation(Node *expr, Oid collation)
case T_CaseExpr:
((CaseExpr *) expr)->casecollid = collation;
break;
+ case T_CaseTestExpr:
+ ((CaseTestExpr *) expr)->collation = collation;
+ break;
case T_ArrayExpr:
((ArrayExpr *) expr)->array_collid = collation;
break;
@@ -1205,6 +1239,33 @@ exprSetCollation(Node *expr, Oid collation)
case T_JsonIsPredicate:
Assert(!OidIsValid(collation)); /* result is always boolean */
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) expr;
+
+ if (jexpr->result_coercion)
+ exprSetCollation((Node *) jexpr->result_coercion, collation);
+ else
+ Assert(!OidIsValid(collation)); /* result is always a
+ * json[b] type */
+ }
+ break;
+ case T_JsonItemCoercion:
+ exprSetCollation((Node *) ((JsonItemCoercion *) expr)->coercion,
+ collation);
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) expr;
+
+ if (coercion->expr)
+ exprSetCollation(coercion->expr, collation);
+ else if (coercion->via_io || coercion->via_populate)
+ coercion->collation = collation;
+ else
+ Assert(!OidIsValid(collation));
+ }
+ break;
case T_NullTest:
/* NullTest's result is boolean ... */
Assert(!OidIsValid(collation)); /* ... so never set a collation */
@@ -1508,6 +1569,15 @@ exprLocation(const Node *expr)
case T_JsonIsPredicate:
loc = ((const JsonIsPredicate *) expr)->location;
break;
+ case T_JsonExpr:
+ {
+ const JsonExpr *jsexpr = (const JsonExpr *) expr;
+
+ /* consider both function name and leftmost arg */
+ loc = leftmostLoc(jsexpr->location,
+ exprLocation(jsexpr->formatted_expr));
+ }
+ break;
case T_NullTest:
{
const NullTest *nexpr = (const NullTest *) expr;
@@ -2260,6 +2330,30 @@ expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ if (WALK(jexpr->formatted_expr))
+ return true;
+ if (WALK(jexpr->result_coercion))
+ return true;
+ if (WALK(jexpr->item_coercions))
+ return true;
+ if (WALK(jexpr->passing_values))
+ return true;
+ /* we assume walker doesn't care about passing_names */
+ if (jexpr->on_empty &&
+ WALK(jexpr->on_empty->default_expr))
+ return true;
+ if (WALK(jexpr->on_error->default_expr))
+ return true;
+ }
+ break;
+ case T_JsonItemCoercion:
+ return WALK(((JsonItemCoercion *) node)->coercion);
+ case T_JsonCoercion:
+ return WALK(((JsonCoercion *) node)->expr);
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
@@ -3259,6 +3353,46 @@ expression_tree_mutator_impl(Node *node,
return (Node *) newnode;
}
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+ JsonExpr *newnode;
+
+ FLATCOPY(newnode, jexpr, JsonExpr);
+ MUTATE(newnode->path_spec, jexpr->path_spec, Node *);
+ MUTATE(newnode->formatted_expr, jexpr->formatted_expr, Node *);
+ MUTATE(newnode->result_coercion, jexpr->result_coercion, JsonCoercion *);
+ MUTATE(newnode->item_coercions, jexpr->item_coercions, List *);
+ MUTATE(newnode->passing_values, jexpr->passing_values, List *);
+ /* assume mutator does not care about passing_names */
+ if (newnode->on_empty)
+ MUTATE(newnode->on_empty->default_expr,
+ jexpr->on_empty->default_expr, Node *);
+ MUTATE(newnode->on_error->default_expr,
+ jexpr->on_error->default_expr, Node *);
+ return (Node *) newnode;
+ }
+ break;
+ case T_JsonItemCoercion:
+ {
+ JsonItemCoercion *item_coercion = (JsonItemCoercion *) node;
+ JsonItemCoercion *newnode;
+
+ FLATCOPY(newnode, item_coercion, JsonItemCoercion);
+ MUTATE(newnode->coercion, item_coercion->coercion, JsonCoercion *);
+ return (Node *) newnode;
+ }
+ break;
+ case T_JsonCoercion:
+ {
+ JsonCoercion *coercion = (JsonCoercion *) node;
+ JsonCoercion *newnode;
+
+ FLATCOPY(newnode, coercion, JsonCoercion);
+ MUTATE(newnode->expr, coercion->expr, Node *);
+ return (Node *) newnode;
+ }
+ break;
case T_NullTest:
{
NullTest *ntest = (NullTest *) node;
@@ -3945,6 +4079,51 @@ raw_expression_tree_walker_impl(Node *node,
break;
case T_JsonIsPredicate:
return WALK(((JsonIsPredicate *) node)->expr);
+ case T_JsonArgument:
+ return WALK(((JsonArgument *) node)->val);
+ case T_JsonCommon:
+ {
+ JsonCommon *jc = (JsonCommon *) node;
+
+ if (WALK(jc->expr))
+ return true;
+ if (WALK(jc->pathspec))
+ return true;
+ if (WALK(jc->passing))
+ return true;
+ }
+ break;
+ case T_JsonBehaviorClause:
+ {
+ JsonBehaviorClause *jbc = (JsonBehaviorClause *) node;
+
+ if (WALK(jbc->on_empty))
+ return true;
+ if (WALK(jbc->on_error))
+ return true;
+ }
+ break;
+ case T_JsonBehavior:
+ {
+ JsonBehavior *jb = (JsonBehavior *) node;
+
+ if (jb->btype == JSON_BEHAVIOR_DEFAULT &&
+ WALK(jb->default_expr))
+ return true;
+ }
+ break;
+ case T_JsonFuncExpr:
+ {
+ JsonFuncExpr *jfe = (JsonFuncExpr *) node;
+
+ if (WALK(jfe->common))
+ return true;
+ if (jfe->output && WALK(jfe->output))
+ return true;
+ if (jfe->behavior)
+ return true;
+ }
+ break;
case T_NullTest:
return WALK(((NullTest *) node)->arg);
case T_BooleanTest:
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index d6ceafd51c..3bb48b64d5 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -4850,7 +4850,8 @@ cost_qual_eval_walker(Node *node, cost_qual_eval_context *context)
IsA(node, SQLValueFunction) ||
IsA(node, XmlExpr) ||
IsA(node, CoerceToDomain) ||
- IsA(node, NextValueExpr))
+ IsA(node, NextValueExpr) ||
+ IsA(node, JsonExpr))
{
/* Treat all these as having cost 1 */
context->total.per_tuple += cpu_operator_cost;
diff --git a/src/backend/optimizer/util/clauses.c b/src/backend/optimizer/util/clauses.c
index 948c2bf06d..64fa79a6f7 100644
--- a/src/backend/optimizer/util/clauses.c
+++ b/src/backend/optimizer/util/clauses.c
@@ -53,6 +53,7 @@
#include "utils/fmgroids.h"
#include "utils/json.h"
#include "utils/jsonb.h"
+#include "utils/jsonpath.h"
#include "utils/lsyscache.h"
#include "utils/memutils.h"
#include "utils/syscache.h"
@@ -412,6 +413,24 @@ contain_mutable_functions_walker(Node *node, void *context)
/* Check all subnodes */
}
+ if (IsA(node, JsonExpr))
+ {
+ JsonExpr *jexpr = castNode(JsonExpr, node);
+ Const *cnst;
+
+ if (!IsA(jexpr->path_spec, Const))
+ return true;
+
+ cnst = castNode(Const, jexpr->path_spec);
+
+ Assert(cnst->consttype == JSONPATHOID);
+ if (cnst->constisnull)
+ return false;
+
+ return jspIsMutable(DatumGetJsonPathP(cnst->constvalue),
+ jexpr->passing_names, jexpr->passing_values);
+ }
+
if (IsA(node, SQLValueFunction))
{
/* all variants of SQLValueFunction are stable */
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index c224df4ecc..1dc3300fde 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -278,6 +278,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
+ JsonBehavior *jsbehavior;
+ JsonBehaviorClause *jsbehaviorclause;
+ JsonQuotes js_quotes;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -650,14 +653,21 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_returning_clause_opt
json_name_and_value
json_aggregate_func
+ json_api_common_syntax
+ json_argument
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
+ json_arguments
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
+ json_wrapper_behavior
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
+%type <jsbehavior> json_behavior
+%type <jsbehaviorclause> json_behavior_clause_opt
+%type <js_quotes> json_quotes_clause_opt
/*
@@ -695,7 +705,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P
CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT
- COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT
+ COMMITTED COMPRESSION CONCURRENTLY CONDITIONAL CONFIGURATION CONFLICT
CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY
COST CREATE CROSS CSV CUBE CURRENT_P
CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA
@@ -706,8 +716,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
DETACH DICTIONARY DISABLE_P DISCARD DISTINCT DO DOCUMENT_P DOMAIN_P
DOUBLE_P DROP
- EACH ELSE ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ESCAPE EVENT EXCEPT
- EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
+ EACH ELSE EMPTY_P ENABLE_P ENCODING ENCRYPTED END_P ENUM_P ERROR_P ESCAPE
+ EVENT EXCEPT EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXISTS EXPLAIN EXPRESSION
EXTENSION EXTERNAL EXTRACT
FALSE_P FAMILY FETCH FILTER FINALIZE FIRST_P FLOAT_P FOLLOWING FOR
@@ -722,10 +732,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER
INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION
- JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_OBJECT JSON_OBJECTAGG
- JSON_SCALAR JSON_SERIALIZE
+ JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
+ JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
- KEY KEYS
+ KEY KEYS KEEP
LABEL LANGUAGE LARGE_P LAST_P LATERAL_P
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
@@ -739,7 +749,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
NULLS_P NUMERIC
- OBJECT_P OF OFF OFFSET OIDS OLD ON ONLY OPERATOR OPTION OPTIONS OR
+ OBJECT_P OF OFF OFFSET OIDS OLD OMIT ON ONLY OPERATOR OPTION OPTIONS OR
ORDER ORDINALITY OTHERS OUT_P OUTER_P
OVER OVERLAPS OVERLAY OVERRIDING OWNED OWNER
@@ -748,7 +758,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
POSITION PRECEDING PRECISION PRESERVE PREPARE PREPARED PRIMARY
PRIOR PRIVILEGES PROCEDURAL PROCEDURE PROCEDURES PROGRAM PUBLICATION
- QUOTE
+ QUOTE QUOTES
RANGE READ REAL REASSIGN RECHECK RECURSIVE REF_P REFERENCES REFERENCING
REFRESH REINDEX RELATIVE_P RELEASE RENAME REPEATABLE REPLACE REPLICA
@@ -759,7 +769,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
SEQUENCE SEQUENCES
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
- START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
+ START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRING_P STRIP_P
SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P SYSTEM_USER
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
@@ -767,7 +777,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
TREAT TRIGGER TRIM TRUE_P
TRUNCATE TRUSTED TYPE_P TYPES_P
- UESCAPE UNBOUNDED UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
+ UESCAPE UNBOUNDED UNCONDITIONAL UNCOMMITTED UNENCRYPTED UNION UNIQUE UNKNOWN
UNLISTEN UNLOGGED UNTIL UPDATE USER USING
VACUUM VALID VALIDATE VALIDATOR VALUE_P VALUES VARCHAR VARIADIC VARYING
@@ -15755,6 +15765,60 @@ func_expr_common_subexpr:
n->location = @1;
$$ = (Node *) n;
}
+ | JSON_QUERY '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_wrapper_behavior
+ json_quotes_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_QUERY_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->wrapper = $5;
+ if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(@6)));
+ n->quotes = $6;
+ n->behavior = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | JSON_EXISTS '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *p = makeNode(JsonFuncExpr);
+
+ p->op = JSON_EXISTS_OP;
+ p->common = (JsonCommon *) $3;
+ p->output = (JsonOutput *) $4;
+ p->behavior = $5;
+ p->location = @1;
+ $$ = (Node *) p;
+ }
+ | JSON_VALUE '('
+ json_api_common_syntax
+ json_returning_clause_opt
+ json_behavior_clause_opt
+ ')'
+ {
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
+
+ n->op = JSON_VALUE_OP;
+ n->common = (JsonCommon *) $3;
+ n->output = (JsonOutput *) $4;
+ n->behavior = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
;
@@ -16481,6 +16545,72 @@ opt_asymmetric: ASYMMETRIC
;
/* SQL/JSON support */
+json_api_common_syntax:
+ json_value_expr ',' a_expr /* i.e. a json_path */
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->passing = NULL;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | json_value_expr ',' a_expr AS name
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->passing = NULL;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+
+ | json_value_expr ',' a_expr /* i.e. a json_path */
+ PASSING json_arguments
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = NULL;
+ n->passing = $5;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ | json_value_expr ',' a_expr AS name
+ PASSING json_arguments
+ {
+ JsonCommon *n = makeNode(JsonCommon);
+
+ n->expr = (JsonValueExpr *) $1;
+ n->pathspec = $3;
+ n->pathname = $5;
+ n->passing = $7;
+ n->location = @1;
+ $$ = (Node *) n;
+ }
+ ;
+
+json_arguments:
+ json_argument { $$ = list_make1($1); }
+ | json_arguments ',' json_argument { $$ = lappend($1, $3); }
+ ;
+
+json_argument:
+ json_value_expr AS ColLabel
+ {
+ JsonArgument *n = makeNode(JsonArgument);
+
+ n->val = (JsonValueExpr *) $1;
+ n->name = $3;
+ $$ = (Node *) n;
+ }
+ ;
+
json_value_expr:
a_expr json_format_clause_opt
{
@@ -16506,6 +16636,27 @@ json_encoding_clause_opt:
| /* EMPTY */ { $$ = JS_ENC_DEFAULT; }
;
+/* ARRAY is a noise word */
+json_wrapper_behavior:
+ WITHOUT WRAPPER { $$ = JSW_NONE; }
+ | WITHOUT ARRAY WRAPPER { $$ = JSW_NONE; }
+ | WITH WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL ARRAY WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL ARRAY WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | WITH CONDITIONAL WRAPPER { $$ = JSW_CONDITIONAL; }
+ | WITH UNCONDITIONAL WRAPPER { $$ = JSW_UNCONDITIONAL; }
+ | /* empty */ { $$ = JSW_NONE; }
+ ;
+
+json_quotes_clause_opt:
+ KEEP QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_KEEP; }
+ | KEEP QUOTES { $$ = JS_QUOTES_KEEP; }
+ | OMIT QUOTES ON SCALAR STRING_P { $$ = JS_QUOTES_OMIT; }
+ | OMIT QUOTES { $$ = JS_QUOTES_OMIT; }
+ | /* EMPTY */ { $$ = JS_QUOTES_UNSPEC; }
+ ;
+
json_returning_clause_opt:
RETURNING Typename json_format_clause_opt
{
@@ -16519,6 +16670,30 @@ json_returning_clause_opt:
| /* EMPTY */ { $$ = NULL; }
;
+json_behavior:
+ DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
+ | ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
+ | TRUE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
+ | FALSE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
+ | UNKNOWN { $$ = makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
+ | EMPTY_P ARRAY { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P OBJECT_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
+ /* non-standard, for Oracle compatibility only */
+ | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ ;
+
+json_behavior_clause_opt:
+ json_behavior ON EMPTY_P
+ { $$ = makeJsonBehaviorClause($1, NULL, @1); }
+ | json_behavior ON ERROR_P
+ { $$ = makeJsonBehaviorClause(NULL, $1, @1); }
+ | json_behavior ON EMPTY_P json_behavior ON ERROR_P
+ { $$ = makeJsonBehaviorClause($1, $4, @1); }
+ | /* EMPTY */
+ { $$ = NULL; }
+ ;
+
json_predicate_type_constraint:
JSON { $$ = JS_TYPE_ANY; }
| JSON VALUE_P { $$ = JS_TYPE_ANY; }
@@ -17108,6 +17283,7 @@ unreserved_keyword:
| COMMIT
| COMMITTED
| COMPRESSION
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17144,10 +17320,12 @@ unreserved_keyword:
| DOUBLE_P
| DROP
| EACH
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17197,6 +17375,7 @@ unreserved_keyword:
| INSTEAD
| INVOKER
| ISOLATION
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17243,6 +17422,7 @@ unreserved_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| OPERATOR
| OPTION
| OPTIONS
@@ -17273,6 +17453,7 @@ unreserved_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REASSIGN
@@ -17332,6 +17513,7 @@ unreserved_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUPPORT
@@ -17354,6 +17536,7 @@ unreserved_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNKNOWN
| UNLISTEN
@@ -17414,10 +17597,13 @@ col_name_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
| LEAST
| NATIONAL
| NCHAR
@@ -17650,6 +17836,7 @@ bare_label_keyword:
| COMMITTED
| COMPRESSION
| CONCURRENTLY
+ | CONDITIONAL
| CONFIGURATION
| CONFLICT
| CONNECTION
@@ -17702,11 +17889,13 @@ bare_label_keyword:
| DROP
| EACH
| ELSE
+ | EMPTY_P
| ENABLE_P
| ENCODING
| ENCRYPTED
| END_P
| ENUM_P
+ | ERROR_P
| ESCAPE
| EVENT
| EXCLUDE
@@ -17776,10 +17965,14 @@ bare_label_keyword:
| JSON
| JSON_ARRAY
| JSON_ARRAYAGG
+ | JSON_EXISTS
| JSON_OBJECT
| JSON_OBJECTAGG
+ | JSON_QUERY
| JSON_SCALAR
| JSON_SERIALIZE
+ | JSON_VALUE
+ | KEEP
| KEY
| KEYS
| LABEL
@@ -17840,6 +18033,7 @@ bare_label_keyword:
| OFF
| OIDS
| OLD
+ | OMIT
| ONLY
| OPERATOR
| OPTION
@@ -17877,6 +18071,7 @@ bare_label_keyword:
| PROGRAM
| PUBLICATION
| QUOTE
+ | QUOTES
| RANGE
| READ
| REAL
@@ -17945,6 +18140,7 @@ bare_label_keyword:
| STORAGE
| STORED
| STRICT_P
+ | STRING_P
| STRIP_P
| SUBSCRIPTION
| SUBSTRING
@@ -17979,6 +18175,7 @@ bare_label_keyword:
| UESCAPE
| UNBOUNDED
| UNCOMMITTED
+ | UNCONDITIONAL
| UNENCRYPTED
| UNIQUE
| UNKNOWN
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64c582c344..64571562c5 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -90,6 +90,23 @@ static Node *transformJsonParseExpr(ParseState *pstate, JsonParseExpr *expr);
static Node *transformJsonScalarExpr(ParseState *pstate, JsonScalarExpr *expr);
static Node *transformJsonSerializeExpr(ParseState *pstate,
JsonSerializeExpr *expr);
+static Node *transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *p);
+static JsonExpr *transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName);
+static void transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names);
+static JsonCoercion *coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr);
+static Oid JsonFuncExprDefaultReturnType(JsonExpr *jsexpr);
+static JsonCoercion *coerceJsonExpr(ParseState *pstate, Node *expr,
+ const JsonReturning *returning);
+static List *InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId);
+static JsonBehavior *transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning);
+static JsonBehavior *coerceJsonBehaviorDefaultExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning);
static Node *make_row_comparison_op(ParseState *pstate, List *opname,
List *largs, List *rargs, int location);
static Node *make_row_distinct_op(ParseState *pstate, List *opname,
@@ -353,6 +370,10 @@ transformExprRecurse(ParseState *pstate, Node *expr)
result = transformJsonSerializeExpr(pstate, (JsonSerializeExpr *) expr);
break;
+ case T_JsonFuncExpr:
+ result = transformJsonFuncExpr(pstate, (JsonFuncExpr *) expr);
+ break;
+
default:
/* should not reach here */
elog(ERROR, "unrecognized node type: %d", (int) nodeTag(expr));
@@ -3229,7 +3250,7 @@ makeJsonByteaToTextConversion(Node *expr, JsonFormat *format, int location)
static Node *
transformJsonValueExpr(ParseState *pstate, const char *constructName,
JsonValueExpr *ve, JsonFormatType default_format,
- Oid targettype)
+ Oid targettype, bool isarg)
{
Node *expr = transformExprRecurse(pstate, (Node *) ve->raw_expr);
Node *rawexpr;
@@ -3261,6 +3282,35 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
else
format = ve->format->format_type;
}
+ else if (isarg)
+ {
+ /* Pass SQL/JSON item types directly without conversion to json[b]. */
+ switch (exprtype)
+ {
+ case TEXTOID:
+ case NUMERICOID:
+ case BOOLOID:
+ case INT2OID:
+ case INT4OID:
+ case INT8OID:
+ case FLOAT4OID:
+ case FLOAT8OID:
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ return expr;
+
+ default:
+ if (typcategory == TYPCATEGORY_STRING)
+ return expr;
+ /* else convert argument to json[b] type */
+ break;
+ }
+
+ format = default_format;
+ }
else if (exprtype == JSONOID || exprtype == JSONBOID)
format = JS_FORMAT_DEFAULT; /* do not format json[b] types */
else
@@ -3272,7 +3322,8 @@ transformJsonValueExpr(ParseState *pstate, const char *constructName,
Node *coerced;
bool only_allow_cast = OidIsValid(targettype);
- if (!only_allow_cast &&
+ if (!isarg &&
+ !only_allow_cast &&
exprtype != BYTEAOID && typcategory != TYPCATEGORY_STRING)
ereport(ERROR,
errcode(ERRCODE_DATATYPE_MISMATCH),
@@ -3425,6 +3476,11 @@ transformJsonOutput(ParseState *pstate, const JsonOutput *output,
errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("returning SETOF types is not supported in SQL/JSON functions"));
+ if (get_typtype(ret->typid) == TYPTYPE_PSEUDO)
+ ereport(ERROR,
+ errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("returning pseudo-types is not supported in SQL/JSON functions"));
+
if (ret->format->format_type == JS_FORMAT_DEFAULT)
/* assign JSONB format when returning jsonb, or JSON format otherwise */
ret->format->format_type =
@@ -3621,7 +3677,7 @@ transformJsonObjectConstructor(ParseState *pstate, JsonObjectConstructor *ctor)
Node *val = transformJsonValueExpr(pstate, "JSON_OBJECT()",
kv->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = lappend(args, key);
args = lappend(args, val);
@@ -3808,7 +3864,7 @@ transformJsonObjectAgg(ParseState *pstate, JsonObjectAgg *agg)
val = transformJsonValueExpr(pstate, "JSON_OBJECTAGG()",
agg->arg->value,
JS_FORMAT_DEFAULT,
- InvalidOid);
+ InvalidOid, false);
args = list_make2(key, val);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
@@ -3864,9 +3920,8 @@ transformJsonArrayAgg(ParseState *pstate, JsonArrayAgg *agg)
Oid aggfnoid;
Oid aggtype;
- arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()",
- agg->arg,
- JS_FORMAT_DEFAULT, InvalidOid);
+ arg = transformJsonValueExpr(pstate, "JSON_ARRAYAGG()", agg->arg,
+ JS_FORMAT_DEFAULT, InvalidOid, false);
returning = transformJsonConstructorOutput(pstate, agg->constructor->output,
list_make1(arg));
@@ -3913,9 +3968,8 @@ transformJsonArrayConstructor(ParseState *pstate, JsonArrayConstructor *ctor)
{
JsonValueExpr *jsval = castNode(JsonValueExpr, lfirst(lc));
Node *val = transformJsonValueExpr(pstate, "JSON_ARRAY()",
- jsval,
- JS_FORMAT_DEFAULT,
- InvalidOid);
+ jsval, JS_FORMAT_DEFAULT,
+ InvalidOid, false);
args = lappend(args, val);
}
@@ -4074,7 +4128,7 @@ transformJsonParseExpr(ParseState *pstate, JsonParseExpr *jsexpr)
* function-like CASTs.
*/
arg = transformJsonValueExpr(pstate, "JSON()", jsexpr->expr,
- JS_FORMAT_JSON, returning->typid);
+ JS_FORMAT_JSON, returning->typid, false);
}
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_PARSE, list_make1(arg), NULL,
@@ -4119,7 +4173,7 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
Node *arg = transformJsonValueExpr(pstate, "JSON_SERIALIZE()",
expr->expr,
JS_FORMAT_JSON,
- InvalidOid);
+ InvalidOid, false);
if (expr->output)
{
@@ -4153,3 +4207,478 @@ transformJsonSerializeExpr(ParseState *pstate, JsonSerializeExpr *expr)
return makeJsonConstructorExpr(pstate, JSCTOR_JSON_SERIALIZE, list_make1(arg),
NULL, returning, false, false, expr->location);
}
+
+/*
+ * Transform JSON_VALUE, JSON_QUERY, JSON_EXISTS functions into a JsonExpr node.
+ */
+static Node *
+transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
+{
+ JsonExpr *jsexpr = NULL;
+ const char *func_name = NULL;
+ JsonBehavior *on_error = NULL,
+ *on_empty = NULL;
+
+ /*
+ * Disallow FORMAT specification in the RETURNING clause of JSON_EXISTS()
+ * and JSON_VALUE().
+ */
+ if (func->output &&
+ (func->op == JSON_VALUE_OP || func->op == JSON_EXISTS_OP))
+ {
+ JsonFormat *format = func->output->returning->format;
+
+ if (format->format_type != JS_FORMAT_DEFAULT ||
+ format->encoding != JS_ENC_DEFAULT)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot specify FORMAT in RETURNING clause of %s",
+ func->op == JSON_VALUE_OP ? "JSON_VALUE()" :
+ "JSON_EXISTS()"),
+ parser_errposition(pstate, format->location)));
+ }
+
+ if (func->behavior)
+ {
+ on_error = func->behavior->on_error;
+ on_empty = func->behavior->on_empty;
+ }
+
+ switch (func->op)
+ {
+ case JSON_EXISTS_OP:
+ func_name = "JSON_EXISTS";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ jsexpr->returning->typid = BOOLOID;
+ jsexpr->returning->typmod = -1;
+ }
+ else if (jsexpr->returning->typid != BOOLOID)
+ {
+ Node *coercion_expr;
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+ int location = exprLocation((Node *) jsexpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the
+ * result of evaluating JSON_EXISTS to the coercion
+ * expression.
+ */
+ placeholder->typeId = BOOLOID;
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ coercion_expr =
+ coerce_to_target_type(pstate, (Node *) placeholder, BOOLOID,
+ jsexpr->returning->typid,
+ jsexpr->returning->typmod,
+ COERCION_EXPLICIT,
+ COERCE_IMPLICIT_CAST,
+ location);
+
+ if (coercion_expr == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast type %s to %s",
+ format_type_be(BOOLOID),
+ format_type_be(jsexpr->returning->typid)),
+ parser_coercion_errposition(pstate, location, (Node *) jsexpr)));
+
+ if (coercion_expr != (Node *) placeholder)
+ {
+ jsexpr->result_coercion = makeNode(JsonCoercion);
+ jsexpr->result_coercion->expr = coercion_expr;
+ }
+ }
+
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_FALSE,
+ jsexpr->returning);
+ break;
+
+ case JSON_QUERY_OP:
+ func_name = "JSON_QUERY";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+ jsexpr->wrapper = func->wrapper;
+ jsexpr->omit_quotes = (func->quotes == JS_QUOTES_OMIT);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ JsonReturning *ret = jsexpr->returning;
+
+ ret->typid = JsonFuncExprDefaultReturnType(jsexpr);
+ ret->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ jsexpr->on_empty = transformJsonBehavior(pstate, on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+
+ case JSON_VALUE_OP:
+ func_name = "JSON_VALUE";
+
+ jsexpr = transformJsonExprCommon(pstate, func, func_name);
+
+ if (!OidIsValid(jsexpr->returning->typid))
+ {
+ /* Make JSON_VALUE return text by default */
+ jsexpr->returning->typid = TEXTOID;
+ jsexpr->returning->typmod = -1;
+ }
+ jsexpr->result_coercion = coerceJsonFuncExprOutput(pstate, jsexpr);
+
+ /*
+ * Initialize JsonCoercion nodes to coerce the scalar value
+ * returned by JsonPathValue() to the "returning" type.
+ */
+ jsexpr->item_coercions =
+ InitJsonItemCoercions(pstate, jsexpr->returning,
+ exprType(jsexpr->formatted_expr));
+
+ jsexpr->on_empty = transformJsonBehavior(pstate, on_empty,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ jsexpr->on_error = transformJsonBehavior(pstate, on_error,
+ JSON_BEHAVIOR_NULL,
+ jsexpr->returning);
+ break;
+ }
+
+ Assert(jsexpr != NULL && jsexpr->formatted_expr != NULL);
+ if (exprType(jsexpr->formatted_expr) != JSONBOID)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("%s() is not yet implemented for the json type", func_name),
+ errhint("Try casting the argument to jsonb"),
+ parser_errposition(pstate, func->location)));
+
+ return (Node *) jsexpr;
+}
+
+/*
+ * Common code for JSON_VALUE, JSON_QUERY, JSON_EXISTS transformation
+ * into a JsonExpr node.
+ */
+static JsonExpr *
+transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
+ const char *constructName)
+{
+ JsonExpr *jsexpr = makeNode(JsonExpr);
+ Node *pathspec;
+
+ jsexpr->location = func->location;
+ jsexpr->op = func->op;
+ jsexpr->formatted_expr = transformJsonValueExpr(pstate, constructName,
+ func->common->expr,
+ JS_FORMAT_JSON,
+ InvalidOid, false);
+
+ jsexpr->format = func->common->expr->format;
+
+ /* Both set in the caller. */
+ jsexpr->result_coercion = NULL;
+ jsexpr->omit_quotes = false;
+
+ pathspec = transformExprRecurse(pstate, func->common->pathspec);
+
+ jsexpr->path_spec =
+ coerce_to_target_type(pstate, pathspec, exprType(pathspec),
+ JSONPATHOID, -1,
+ COERCION_EXPLICIT, COERCE_IMPLICIT_CAST,
+ exprLocation(pathspec));
+ if (!jsexpr->path_spec)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("JSON path expression must be of type %s, not of type %s",
+ "jsonpath", format_type_be(exprType(pathspec))),
+ parser_errposition(pstate, exprLocation(pathspec))));
+
+ /*
+ * Transform and coerce to json[b] passing arguments, whose format is
+ * determined by context item type.
+ */
+ transformJsonPassingArgs(pstate, constructName,
+ exprType(jsexpr->formatted_expr) == JSONBOID ?
+ JS_FORMAT_JSONB : JS_FORMAT_JSON,
+ func->common->passing,
+ &jsexpr->passing_values,
+ &jsexpr->passing_names);
+
+ jsexpr->returning = transformJsonOutput(pstate, func->output, false);
+
+ /* JSON_QUERY support specifying FORMAT explicitly. */
+ if (func->op != JSON_QUERY_OP)
+ {
+ jsexpr->returning->format->format_type = JS_FORMAT_DEFAULT;
+ jsexpr->returning->format->encoding = JS_ENC_DEFAULT;
+ }
+
+ return jsexpr;
+}
+
+/*
+ * Transform a JSON PASSING clause.
+ */
+static void
+transformJsonPassingArgs(ParseState *pstate, const char *constructName,
+ JsonFormatType format, List *args,
+ List **passing_values, List **passing_names)
+{
+ ListCell *lc;
+
+ *passing_values = NIL;
+ *passing_names = NIL;
+
+ foreach(lc, args)
+ {
+ JsonArgument *arg = castNode(JsonArgument, lfirst(lc));
+ Node *expr = transformJsonValueExpr(pstate, constructName,
+ arg->val, format,
+ InvalidOid, true);
+
+ *passing_values = lappend(*passing_values, expr);
+ *passing_names = lappend(*passing_names, makeString(arg->name));
+ }
+}
+
+/*
+ * Transform the JSON output clause of JSON_VALUE and JSON_QUERY.
+ */
+static JsonCoercion *
+coerceJsonFuncExprOutput(ParseState *pstate, JsonExpr *jsexpr)
+{
+ JsonCoercion *coercion = NULL;
+ JsonReturning *returning = jsexpr->returning;
+ Node *context_item = jsexpr->formatted_expr;
+ int default_typmod;
+ Oid default_typid;
+
+ Assert(returning);
+
+ /*
+ * Forced coercion via I/O for non-JSON types, except for JSON_QUERY()
+ * which must implement the specified QUORES or WRAPPER behavior.
+ */
+ if (jsexpr->op == JSON_QUERY_OP &&
+ returning->typid != JSONOID &&
+ returning->typid != JSONBOID &&
+ (jsexpr->omit_quotes || jsexpr->wrapper != JSW_NONE))
+ {
+ coercion = makeNode(JsonCoercion);
+ coercion->expr = NULL;
+ coercion->via_io = jsexpr->omit_quotes;
+ coercion->via_populate = jsexpr->wrapper != JSW_NONE;
+
+ return coercion;
+ }
+
+ default_typid = JsonFuncExprDefaultReturnType(jsexpr);
+ default_typmod = -1;
+ if (returning->typid != default_typid ||
+ returning->typmod != default_typmod)
+ {
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * evaluating the JSON_VALUE/QUERY jsonpath expression to the coercion
+ * function.
+ */
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ placeholder->typeId = exprType(context_item);
+ placeholder->typeMod = exprTypmod(context_item);
+ placeholder->collation = exprCollation(context_item);
+
+ Assert(placeholder->typeId == default_typid);
+ Assert(placeholder->typeMod == default_typmod);
+
+ coercion = coerceJsonExpr(pstate, (Node *) placeholder, returning);
+ }
+
+ return coercion;
+}
+
+/* Returns the default type for a given JsonExpr for a given JsonFormat. */
+static Oid
+JsonFuncExprDefaultReturnType(JsonExpr *jsexpr)
+{
+ JsonFormat *format = jsexpr->format;
+ Node *context_item = jsexpr->formatted_expr;
+
+ Assert(format);
+ if (format->format_type == JS_FORMAT_JSONB)
+ return JSONBOID;
+ else if (format->format_type == JS_FORMAT_DEFAULT &&
+ exprType(context_item) == JSONBOID)
+ return JSONBOID;
+
+ return JSONOID;
+}
+
+/*
+ * Set up a JsonCoercion node to:
+ *
+ * - coerce expression to the output returning type, or
+ * - coerce using json_populate_type() if returning type requires it, or
+ * - coerce via I/O.
+ */
+static JsonCoercion *
+coerceJsonExpr(ParseState *pstate, Node *expr, const JsonReturning *returning)
+{
+ char typtype;
+ JsonCoercion *coercion = makeNode(JsonCoercion);
+
+ coercion->expr = coerceJsonFuncExpr(pstate, expr, returning, false);
+ if (coercion->expr)
+ {
+ if (coercion->expr == expr)
+ coercion->expr = NULL;
+
+ return coercion;
+ }
+
+ typtype = get_typtype(returning->typid);
+ if (returning->typid == RECORDOID ||
+ typtype == TYPTYPE_COMPOSITE ||
+ typtype == TYPTYPE_DOMAIN ||
+ type_is_array(returning->typid))
+ coercion->via_populate = true;
+ else
+ coercion->via_io = true;
+
+ return coercion;
+}
+
+/*
+ * Initialize JsonCoercion nodes for coercing a given JSON item value produced
+ * by JSON_VALUE to the target "returning" type; also see
+ * ExecPrepareJsonItemCoercion().
+ */
+static List *
+InitJsonItemCoercions(ParseState *pstate, const JsonReturning *returning,
+ Oid contextItemTypeId)
+{
+ List *item_coercions = NIL;
+ int i;
+ Oid typeoid;
+ struct
+ {
+ JsonItemType item_type;
+ Oid typeoid;
+ } item_types[] =
+ {
+ {JsonItemTypeNull, UNKNOWNOID},
+ {JsonItemTypeString, TEXTOID},
+ {JsonItemTypeNumeric, NUMERICOID},
+ {JsonItemTypeBoolean, BOOLOID},
+ {JsonItemTypeDate, DATEOID},
+ {JsonItemTypeTime, TIMEOID},
+ {JsonItemTypeTimetz, TIMETZOID},
+ {JsonItemTypeTimestamp, TIMESTAMPOID},
+ {JsonItemTypeTimestamptz, TIMESTAMPTZOID},
+ {JsonItemTypeComposite, contextItemTypeId},
+ {JsonItemTypeInvalid, InvalidOid}
+ };
+
+ for (i = 0; OidIsValid(typeoid = item_types[i].typeoid); i++)
+ {
+ Node *expr;
+ JsonItemCoercion *item_coercion = makeNode(JsonItemCoercion);
+
+ if (typeoid == UNKNOWNOID)
+ {
+ expr = (Node *) makeNullConst(UNKNOWNOID, -1, InvalidOid);
+ }
+ else
+ {
+ CaseTestExpr *placeholder = makeNode(CaseTestExpr);
+
+ /*
+ * We abuse CaseTestExpr here as placeholder to pass the result of
+ * JSON_VALUE jsonpath expression to the coercion function.
+ */
+ placeholder->typeId = item_types[i].typeoid;
+ placeholder->typeMod = -1;
+ placeholder->collation = InvalidOid;
+
+ expr = (Node *) placeholder;
+ }
+
+ item_coercion->item_type = item_types[i].item_type;
+ item_coercion->coercion = coerceJsonExpr(pstate, expr, returning);
+ item_coercions = lappend(item_coercions, item_coercion);
+ }
+
+ return item_coercions;
+}
+
+/*
+ * Transform a JSON BEHAVIOR clause.
+ */
+static JsonBehavior *
+transformJsonBehavior(ParseState *pstate, JsonBehavior *behavior,
+ JsonBehaviorType default_behavior,
+ JsonReturning *returning)
+{
+ JsonBehaviorType behavior_type = default_behavior;
+ Node *default_expr = NULL;
+
+ if (behavior)
+ {
+ behavior_type = behavior->btype;
+ if (behavior_type == JSON_BEHAVIOR_DEFAULT)
+ default_expr = transformExprRecurse(pstate, behavior->default_expr);
+ }
+
+ behavior = makeJsonBehavior(behavior_type, default_expr, -1);
+
+ /*
+ * Also coerce the DEFAULT expression, if any, to match the returning
+ * type.
+ */
+ return coerceJsonBehaviorDefaultExpr(pstate, behavior, returning);
+}
+
+/*
+ * Coerce expression in `DEFAULT expression ON ERROR / EMPTY` to the target
+ * output type.
+ */
+static JsonBehavior *
+coerceJsonBehaviorDefaultExpr(ParseState *pstate, JsonBehavior *behavior,
+ JsonReturning *returning)
+{
+ Oid exprtype;
+
+ if (behavior->default_expr == NULL)
+ return behavior;
+
+ exprtype = exprType(behavior->default_expr);
+
+ behavior->default_expr =
+ coerce_to_target_type(pstate,
+ behavior->default_expr,
+ exprtype,
+ returning->typid,
+ returning->typmod,
+ COERCION_EXPLICIT,
+ COERCE_IMPLICIT_CAST,
+ exprLocation((Node *) behavior));
+
+ if (behavior->default_expr == NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_CANNOT_COERCE),
+ errmsg("cannot cast DEFAULT expression of type %s to %s",
+ format_type_be(exprtype),
+ format_type_be(returning->typid)),
+ parser_errposition(pstate, exprLocation((Node *) behavior))));
+
+ return behavior;
+}
diff --git a/src/backend/parser/parse_target.c b/src/backend/parser/parse_target.c
index 3bc62ac3ba..8f81624f97 100644
--- a/src/backend/parser/parse_target.c
+++ b/src/backend/parser/parse_target.c
@@ -1989,6 +1989,21 @@ FigureColnameInternal(Node *node, char **name)
/* make JSON_ARRAYAGG act like a regular function */
*name = "json_arrayagg";
return 2;
+ case T_JsonFuncExpr:
+ /* make SQL/JSON functions act like a regular function */
+ switch (((JsonFuncExpr *) node)->op)
+ {
+ case JSON_EXISTS_OP:
+ *name = "json_exists";
+ return 2;
+ case JSON_QUERY_OP:
+ *name = "json_query";
+ return 2;
+ case JSON_VALUE_OP:
+ *name = "json_value";
+ return 2;
+ }
+ break;
default:
break;
}
diff --git a/src/backend/utils/adt/formatting.c b/src/backend/utils/adt/formatting.c
index 8131091f79..e5451c47a2 100644
--- a/src/backend/utils/adt/formatting.c
+++ b/src/backend/utils/adt/formatting.c
@@ -4426,6 +4426,50 @@ parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
}
}
+/*
+ * Parses the datetime format string in 'fmt_str' and returns true if it
+ * contains a timezone specifier, false if not.
+ */
+bool
+datetime_format_has_tz(const char *fmt_str)
+{
+ bool incache;
+ int fmt_len = strlen(fmt_str);
+ int result;
+ FormatNode *format;
+
+ if (fmt_len > DCH_CACHE_SIZE)
+ {
+ /*
+ * Allocate new memory if format picture is bigger than static cache
+ * and do not use cache (call parser always)
+ */
+ incache = false;
+
+ format = (FormatNode *) palloc((fmt_len + 1) * sizeof(FormatNode));
+
+ parse_format(format, fmt_str, DCH_keywords,
+ DCH_suff, DCH_index, DCH_FLAG, NULL);
+ }
+ else
+ {
+ /*
+ * Use cache buffers
+ */
+ DCHCacheEntry *ent = DCH_cache_fetch(fmt_str, false);
+
+ incache = true;
+ format = ent->format;
+ }
+
+ result = DCH_datetime_type(format);
+
+ if (!incache)
+ pfree(format);
+
+ return result & DCH_ZONED;
+}
+
/*
* do_to_timestamp: shared code for to_timestamp and to_date
*
diff --git a/src/backend/utils/adt/jsonb.c b/src/backend/utils/adt/jsonb.c
index 6f445f5c2b..6c255402c4 100644
--- a/src/backend/utils/adt/jsonb.c
+++ b/src/backend/utils/adt/jsonb.c
@@ -2163,3 +2163,65 @@ jsonb_float8(PG_FUNCTION_ARGS)
PG_RETURN_DATUM(retValue);
}
+
+/*
+ * Construct an empty array jsonb.
+ */
+Jsonb *
+JsonbMakeEmptyArray(void)
+{
+ JsonbValue jbv;
+
+ jbv.type = jbvArray;
+ jbv.val.array.elems = NULL;
+ jbv.val.array.nElems = 0;
+ jbv.val.array.rawScalar = false;
+
+ return JsonbValueToJsonb(&jbv);
+}
+
+/*
+ * Construct an empty object jsonb.
+ */
+Jsonb *
+JsonbMakeEmptyObject(void)
+{
+ JsonbValue jbv;
+
+ jbv.type = jbvObject;
+ jbv.val.object.pairs = NULL;
+ jbv.val.object.nPairs = 0;
+
+ return JsonbValueToJsonb(&jbv);
+}
+
+/*
+ * Convert jsonb to a C-string stripping quotes from scalar strings.
+ */
+char *
+JsonbUnquote(Jsonb *jb)
+{
+ if (JB_ROOT_IS_SCALAR(jb))
+ {
+ JsonbValue v;
+
+ (void) JsonbExtractScalar(&jb->root, &v);
+
+ if (v.type == jbvString)
+ return pnstrdup(v.val.string.val, v.val.string.len);
+ else if (v.type == jbvBool)
+ return pstrdup(v.val.boolean ? "true" : "false");
+ else if (v.type == jbvNumeric)
+ return DatumGetCString(DirectFunctionCall1(numeric_out,
+ PointerGetDatum(v.val.numeric)));
+ else if (v.type == jbvNull)
+ return pstrdup("null");
+ else
+ {
+ elog(ERROR, "unrecognized jsonb value type %d", v.type);
+ return NULL;
+ }
+ }
+ else
+ return JsonbToCString(NULL, &jb->root, VARSIZE(jb));
+}
diff --git a/src/backend/utils/adt/jsonfuncs.c b/src/backend/utils/adt/jsonfuncs.c
index 1574ed5985..b495aedce1 100644
--- a/src/backend/utils/adt/jsonfuncs.c
+++ b/src/backend/utils/adt/jsonfuncs.c
@@ -2805,7 +2805,8 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
check_stack_depth();
- if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc))
+ if (jbv->type != jbvBinary || !JsonContainerIsArray(jbc) ||
+ JsonContainerIsScalar(jbc))
{
populate_array_report_expected_array(ctx, ndim - 1);
/* Getting here means the error was reported softly. */
@@ -2813,8 +2814,6 @@ populate_array_dim_jsonb(PopulateArrayContext *ctx, /* context */
return false;
}
- Assert(!JsonContainerIsScalar(jbc));
-
it = JsonbIteratorInit(jbc);
tok = JsonbIteratorNext(&it, &val, true);
@@ -3349,6 +3348,53 @@ populate_record_field(ColumnIOData *col,
}
}
+/* recursively populate specified type from a json/jsonb value */
+Datum
+json_populate_type(Datum json_val, Oid json_type, Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext)
+{
+ JsValue jsv = {0};
+ JsonbValue jbv;
+
+ jsv.is_json = json_type == JSONOID;
+
+ if (*isnull)
+ {
+ if (jsv.is_json)
+ jsv.val.json.str = NULL;
+ else
+ jsv.val.jsonb = NULL;
+ }
+ else if (jsv.is_json)
+ {
+ text *json = DatumGetTextPP(json_val);
+
+ jsv.val.json.str = VARDATA_ANY(json);
+ jsv.val.json.len = VARSIZE_ANY_EXHDR(json);
+ jsv.val.json.type = JSON_TOKEN_INVALID; /* not used in
+ * populate_composite() */
+ }
+ else
+ {
+ Jsonb *jsonb = DatumGetJsonbP(json_val);
+
+ jsv.val.jsonb = &jbv;
+
+ /* fill binary jsonb value pointing to jb */
+ jbv.type = jbvBinary;
+ jbv.val.binary.data = &jsonb->root;
+ jbv.val.binary.len = VARSIZE(jsonb) - VARHDRSZ;
+ }
+
+ if (!*cache)
+ *cache = MemoryContextAllocZero(mcxt, sizeof(ColumnIOData));
+
+ return populate_record_field(*cache, typid, typmod, NULL, mcxt,
+ PointerGetDatum(NULL), &jsv, isnull,
+ escontext);
+}
+
static RecordIOData *
allocate_record_info(MemoryContext mcxt, int ncolumns)
{
diff --git a/src/backend/utils/adt/jsonpath.c b/src/backend/utils/adt/jsonpath.c
index c5ba3b7f1d..8d89994c10 100644
--- a/src/backend/utils/adt/jsonpath.c
+++ b/src/backend/utils/adt/jsonpath.c
@@ -68,7 +68,9 @@
#include "libpq/pqformat.h"
#include "nodes/miscnodes.h"
#include "miscadmin.h"
+#include "nodes/nodeFuncs.h"
#include "utils/builtins.h"
+#include "utils/formatting.h"
#include "utils/json.h"
#include "utils/jsonpath.h"
@@ -1110,3 +1112,256 @@ jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from, JsonPathItem *to,
return true;
}
+
+/* SQL/JSON datatype status: */
+typedef enum JsonPathDatatypeStatus
+{
+ jpdsNonDateTime, /* null, bool, numeric, string, array, object */
+ jpdsUnknownDateTime, /* unknown datetime type */
+ jpdsDateTimeZoned, /* timetz, timestamptz */
+ jpdsDateTimeNonZoned /* time, timestamp, date */
+} JsonPathDatatypeStatus;
+
+/* Context for jspIsMutableWalker() */
+typedef struct JsonPathMutableContext
+{
+ List *varnames; /* list of variable names */
+ List *varexprs; /* list of variable expressions */
+ JsonPathDatatypeStatus current; /* status of @ item */
+ bool lax; /* jsonpath is lax or strict */
+ bool mutable; /* resulting mutability status */
+} JsonPathMutableContext;
+
+/*
+ * Recursive walker for jspIsMutable()
+ */
+static JsonPathDatatypeStatus
+jspIsMutableWalker(JsonPathItem *jpi, JsonPathMutableContext *cxt)
+{
+ JsonPathItem next;
+ JsonPathDatatypeStatus status = jpdsNonDateTime;
+
+ while (!cxt->mutable)
+ {
+ JsonPathItem arg;
+ JsonPathDatatypeStatus leftStatus;
+ JsonPathDatatypeStatus rightStatus;
+
+ switch (jpi->type)
+ {
+ case jpiRoot:
+ Assert(status == jpdsNonDateTime);
+ break;
+
+ case jpiCurrent:
+ Assert(status == jpdsNonDateTime);
+ status = cxt->current;
+ break;
+
+ case jpiFilter:
+ {
+ JsonPathDatatypeStatus prevStatus = cxt->current;
+
+ cxt->current = status;
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+
+ cxt->current = prevStatus;
+ break;
+ }
+
+ case jpiVariable:
+ {
+ int32 len;
+ const char *name = jspGetString(jpi, &len);
+ ListCell *lc1;
+ ListCell *lc2;
+
+ Assert(status == jpdsNonDateTime);
+
+ forboth(lc1, cxt->varnames, lc2, cxt->varexprs)
+ {
+ String *varname = lfirst_node(String, lc1);
+ Node *varexpr = lfirst(lc2);
+
+ if (strncmp(varname->sval, name, len))
+ continue;
+
+ switch (exprType(varexpr))
+ {
+ case DATEOID:
+ case TIMEOID:
+ case TIMESTAMPOID:
+ status = jpdsDateTimeNonZoned;
+ break;
+
+ case TIMETZOID:
+ case TIMESTAMPTZOID:
+ status = jpdsDateTimeZoned;
+ break;
+
+ default:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ break;
+ }
+ break;
+ }
+
+ case jpiEqual:
+ case jpiNotEqual:
+ case jpiLess:
+ case jpiGreater:
+ case jpiLessOrEqual:
+ case jpiGreaterOrEqual:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ leftStatus = jspIsMutableWalker(&arg, cxt);
+
+ jspGetRightArg(jpi, &arg);
+ rightStatus = jspIsMutableWalker(&arg, cxt);
+
+ /*
+ * Comparison of datetime type with different timezone status
+ * is mutable.
+ */
+ if (leftStatus != jpdsNonDateTime &&
+ rightStatus != jpdsNonDateTime &&
+ (leftStatus == jpdsUnknownDateTime ||
+ rightStatus == jpdsUnknownDateTime ||
+ leftStatus != rightStatus))
+ cxt->mutable = true;
+ break;
+
+ case jpiNot:
+ case jpiIsUnknown:
+ case jpiExists:
+ case jpiPlus:
+ case jpiMinus:
+ Assert(status == jpdsNonDateTime);
+ jspGetArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiAnd:
+ case jpiOr:
+ case jpiAdd:
+ case jpiSub:
+ case jpiMul:
+ case jpiDiv:
+ case jpiMod:
+ case jpiStartsWith:
+ Assert(status == jpdsNonDateTime);
+ jspGetLeftArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ jspGetRightArg(jpi, &arg);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ case jpiIndexArray:
+ for (int i = 0; i < jpi->content.array.nelems; i++)
+ {
+ JsonPathItem from;
+ JsonPathItem to;
+
+ if (jspGetArraySubscript(jpi, &from, &to, i))
+ jspIsMutableWalker(&to, cxt);
+
+ jspIsMutableWalker(&from, cxt);
+ }
+ /* FALLTHROUGH */
+
+ case jpiAnyArray:
+ if (!cxt->lax)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiAny:
+ if (jpi->content.anybounds.first > 0)
+ status = jpdsNonDateTime;
+ break;
+
+ case jpiDatetime:
+ if (jpi->content.arg)
+ {
+ char *template;
+
+ jspGetArg(jpi, &arg);
+ if (arg.type != jpiString)
+ {
+ status = jpdsNonDateTime;
+ break; /* there will be runtime error */
+ }
+
+ template = jspGetString(&arg, NULL);
+ if (datetime_format_has_tz(template))
+ status = jpdsDateTimeZoned;
+ else
+ status = jpdsDateTimeNonZoned;
+ }
+ else
+ {
+ status = jpdsUnknownDateTime;
+ }
+ break;
+
+ case jpiLikeRegex:
+ Assert(status == jpdsNonDateTime);
+ jspInitByBuffer(&arg, jpi->base, jpi->content.like_regex.expr);
+ jspIsMutableWalker(&arg, cxt);
+ break;
+
+ /* literals */
+ case jpiNull:
+ case jpiString:
+ case jpiNumeric:
+ case jpiBool:
+ /* accessors */
+ case jpiKey:
+ case jpiAnyKey:
+ /* special items */
+ case jpiSubscript:
+ case jpiLast:
+ /* item methods */
+ case jpiType:
+ case jpiSize:
+ case jpiAbs:
+ case jpiFloor:
+ case jpiCeiling:
+ case jpiDouble:
+ case jpiKeyValue:
+ status = jpdsNonDateTime;
+ break;
+ }
+
+ if (!jspGetNext(jpi, &next))
+ break;
+
+ jpi = &next;
+ }
+
+ return status;
+}
+
+/*
+ * Check whether jsonpath expression is immutable or not.
+ */
+bool
+jspIsMutable(JsonPath *path, List *varnames, List *varexprs)
+{
+ JsonPathMutableContext cxt;
+ JsonPathItem jpi;
+
+ cxt.varnames = varnames;
+ cxt.varexprs = varexprs;
+ cxt.current = jpdsNonDateTime;
+ cxt.lax = (path->header & JSONPATH_LAX) != 0;
+ cxt.mutable = false;
+
+ jspInit(&jpi, path);
+ jspIsMutableWalker(&jpi, &cxt);
+
+ return cxt.mutable;
+}
diff --git a/src/backend/utils/adt/jsonpath_exec.c b/src/backend/utils/adt/jsonpath_exec.c
index 2d0599b4aa..6c5602c64d 100644
--- a/src/backend/utils/adt/jsonpath_exec.c
+++ b/src/backend/utils/adt/jsonpath_exec.c
@@ -87,12 +87,16 @@ typedef struct JsonBaseObjectInfo
int id;
} JsonBaseObjectInfo;
+typedef int (*JsonPathVarCallback) (void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
+
/*
* Context of jsonpath execution.
*/
typedef struct JsonPathExecContext
{
- Jsonb *vars; /* variables to substitute into jsonpath */
+ void *vars; /* variables to substitute into jsonpath */
+ JsonPathVarCallback getVar;
JsonbValue *root; /* for $ evaluation */
JsonbValue *current; /* for @ evaluation */
JsonBaseObjectInfo baseObject; /* "base object" for .keyvalue()
@@ -174,7 +178,8 @@ typedef JsonPathBool (*JsonPathPredicateCallback) (JsonPathItem *jsp,
void *param);
typedef Numeric (*BinaryArithmFunc) (Numeric num1, Numeric num2, bool *error);
-static JsonPathExecResult executeJsonPath(JsonPath *path, Jsonb *vars,
+static JsonPathExecResult executeJsonPath(JsonPath *path, void *vars,
+ JsonPathVarCallback getVar,
Jsonb *json, bool throwErrors,
JsonValueList *result, bool useTz);
static JsonPathExecResult executeItem(JsonPathExecContext *cxt,
@@ -225,8 +230,13 @@ static JsonPathExecResult appendBoolResult(JsonPathExecContext *cxt,
JsonPathItem *jsp, JsonValueList *found, JsonPathBool res);
static void getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
JsonbValue *value);
+static int GetJsonPathVar(void *vars, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject);
static void getJsonPathVariable(JsonPathExecContext *cxt,
- JsonPathItem *variable, Jsonb *vars, JsonbValue *value);
+ JsonPathItem *variable, JsonbValue *value);
+static int getJsonPathVariableFromJsonb(void *varsJsonb, char *varName,
+ int varNameLen, JsonbValue *val,
+ JsonbValue *baseObject);
static int JsonbArraySize(JsonbValue *jb);
static JsonPathBool executeComparison(JsonPathItem *cmp, JsonbValue *lv,
JsonbValue *rv, void *p);
@@ -284,7 +294,8 @@ jsonb_path_exists_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- res = executeJsonPath(jp, vars, jb, !silent, NULL, tz);
+ res = executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, NULL, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -339,7 +350,8 @@ jsonb_path_match_internal(FunctionCallInfo fcinfo, bool tz)
silent = PG_GETARG_BOOL(3);
}
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_FREE_IF_COPY(jb, 0);
PG_FREE_IF_COPY(jp, 1);
@@ -417,7 +429,8 @@ jsonb_path_query_internal(FunctionCallInfo fcinfo, bool tz)
vars = PG_GETARG_JSONB_P_COPY(2);
silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
funcctx->user_fctx = JsonValueListGetList(&found);
@@ -464,7 +477,8 @@ jsonb_path_query_array_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
PG_RETURN_JSONB_P(JsonbValueToJsonb(wrapItemsInArray(&found)));
}
@@ -495,7 +509,8 @@ jsonb_path_query_first_internal(FunctionCallInfo fcinfo, bool tz)
Jsonb *vars = PG_GETARG_JSONB_P(2);
bool silent = PG_GETARG_BOOL(3);
- (void) executeJsonPath(jp, vars, jb, !silent, &found, tz);
+ (void) executeJsonPath(jp, vars, getJsonPathVariableFromJsonb,
+ jb, !silent, &found, tz);
if (JsonValueListLength(&found) >= 1)
PG_RETURN_JSONB_P(JsonbValueToJsonb(JsonValueListHead(&found)));
@@ -537,8 +552,9 @@ jsonb_path_query_first_tz(PG_FUNCTION_ARGS)
* In other case it tries to find all the satisfied result items.
*/
static JsonPathExecResult
-executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
- JsonValueList *result, bool useTz)
+executeJsonPath(JsonPath *path, void *vars, JsonPathVarCallback getVar,
+ Jsonb *json, bool throwErrors, JsonValueList *result,
+ bool useTz)
{
JsonPathExecContext cxt;
JsonPathExecResult res;
@@ -550,22 +566,16 @@ executeJsonPath(JsonPath *path, Jsonb *vars, Jsonb *json, bool throwErrors,
if (!JsonbExtractScalar(&json->root, &jbv))
JsonbInitBinary(&jbv, json);
- if (vars && !JsonContainerIsObject(&vars->root))
- {
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
- errmsg("\"vars\" argument is not an object"),
- errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
- }
-
cxt.vars = vars;
+ cxt.getVar = getVar;
cxt.laxMode = (path->header & JSONPATH_LAX) != 0;
cxt.ignoreStructuralErrors = cxt.laxMode;
cxt.root = &jbv;
cxt.current = &jbv;
cxt.baseObject.jbc = NULL;
cxt.baseObject.id = 0;
- cxt.lastGeneratedObjectId = vars ? 2 : 1;
+ /* 1 + number of base objects in vars */
+ cxt.lastGeneratedObjectId = 1 + getVar(vars, NULL, 0, NULL, NULL);
cxt.innermostArraySize = -1;
cxt.throwErrors = throwErrors;
cxt.useTz = useTz;
@@ -2108,54 +2118,118 @@ getJsonPathItem(JsonPathExecContext *cxt, JsonPathItem *item,
&value->val.string.len);
break;
case jpiVariable:
- getJsonPathVariable(cxt, item, cxt->vars, value);
+ getJsonPathVariable(cxt, item, value);
return;
default:
elog(ERROR, "unexpected jsonpath item type");
}
}
+/*
+ * Returns the computed value of a JSON path variable with given name.
+ */
+static int
+GetJsonPathVar(void *cxt, char *varName, int varNameLen,
+ JsonbValue *val, JsonbValue *baseObject)
+{
+ JsonPathVariable *var = NULL;
+ List *vars = cxt;
+ ListCell *lc;
+ int id = 1;
+
+ if (!varName)
+ return list_length(vars);
+
+ foreach(lc, vars)
+ {
+ JsonPathVariable *curvar = lfirst(lc);
+
+ if (!strncmp(curvar->name, varName, varNameLen))
+ {
+ var = curvar;
+ break;
+ }
+
+ id++;
+ }
+
+ if (!var)
+ return -1;
+
+ if (var->isnull)
+ {
+ val->type = jbvNull;
+ return 0;
+ }
+
+ JsonItemFromDatum(var->value, var->typid, var->typmod, val);
+
+ *baseObject = *val;
+ return id;
+}
+
/*
* Get the value of variable passed to jsonpath executor
*/
static void
getJsonPathVariable(JsonPathExecContext *cxt, JsonPathItem *variable,
- Jsonb *vars, JsonbValue *value)
+ JsonbValue *value)
{
char *varName;
int varNameLength;
+ JsonbValue baseObject;
+ int baseObjectId;
+
+ Assert(variable->type == jpiVariable);
+ varName = jspGetString(variable, &varNameLength);
+
+ if (!cxt->vars ||
+ (baseObjectId = cxt->getVar(cxt->vars, varName, varNameLength, value,
+ &baseObject)) < 0)
+ ereport(ERROR,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("could not find jsonpath variable \"%s\"",
+ pnstrdup(varName, varNameLength))));
+
+ if (baseObjectId > 0)
+ setBaseObject(cxt, &baseObject, baseObjectId);
+}
+
+static int
+getJsonPathVariableFromJsonb(void *varsJsonb, char *varName, int varNameLength,
+ JsonbValue *value, JsonbValue *baseObject)
+{
+ Jsonb *vars = varsJsonb;
JsonbValue tmp;
JsonbValue *v;
- if (!vars)
+ if (!varName)
{
- value->type = jbvNull;
- return;
+ if (vars && !JsonContainerIsObject(&vars->root))
+ {
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("\"vars\" argument is not an object"),
+ errdetail("Jsonpath parameters should be encoded as key-value pairs of \"vars\" object.")));
+ }
+
+ return vars ? 1 : 0; /* count of base objects */
}
- Assert(variable->type == jpiVariable);
- varName = jspGetString(variable, &varNameLength);
tmp.type = jbvString;
tmp.val.string.val = varName;
tmp.val.string.len = varNameLength;
v = findJsonbValueFromContainer(&vars->root, JB_FOBJECT, &tmp);
- if (v)
- {
- *value = *v;
- pfree(v);
- }
- else
- {
- ereport(ERROR,
- (errcode(ERRCODE_UNDEFINED_OBJECT),
- errmsg("could not find jsonpath variable \"%s\"",
- pnstrdup(varName, varNameLength))));
- }
+ if (!v)
+ return -1;
- JsonbInitBinary(&tmp, vars);
- setBaseObject(cxt, &tmp, 1);
+ *value = *v;
+ pfree(v);
+
+ JsonbInitBinary(baseObject, vars);
+ return 1;
}
/**************** Support functions for JsonPath execution *****************/
@@ -2812,3 +2886,240 @@ compareDatetime(Datum val1, Oid typid1, Datum val2, Oid typid2,
return DatumGetInt32(DirectFunctionCall2(cmpfunc, val1, val2));
}
+
+/* Executor-callable JSON_EXISTS implementation */
+bool
+JsonPathExists(Datum jb, JsonPath *jp, bool *error, List *vars)
+{
+ JsonPathExecResult res = executeJsonPath(jp, vars, GetJsonPathVar,
+ DatumGetJsonbP(jb), !error, NULL,
+ true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ *error = true;
+
+ return res == jperOk;
+}
+
+/* Executor-callable JSON_QUERY implementation */
+Datum
+JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper, bool *empty,
+ bool *error, List *vars)
+{
+ JsonbValue *first;
+ bool wrap;
+ JsonValueList found = {0};
+ JsonPathExecResult res PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ res = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(res));
+
+ if (error && jperIsError(res))
+ {
+ *error = true;
+ *empty = false;
+ return (Datum) 0;
+ }
+
+ count = JsonValueListLength(&found);
+
+ first = count > 0 ? JsonValueListHead(&found) : NULL;
+
+ if (!first)
+ wrap = false;
+ else if (wrapper == JSW_NONE)
+ wrap = false;
+ else if (wrapper == JSW_UNCONDITIONAL)
+ wrap = true;
+ else if (wrapper == JSW_CONDITIONAL)
+ wrap = count > 1 ||
+ IsAJsonbScalar(first) ||
+ (first->type == jbvBinary &&
+ JsonContainerIsScalar(first->val.binary.data));
+ else
+ {
+ elog(ERROR, "unrecognized json wrapper %d", wrapper);
+ wrap = false;
+ }
+
+ if (wrap)
+ return JsonbPGetDatum(JsonbValueToJsonb(wrapItemsInArray(&found)));
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return (Datum) 0;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_QUERY should return singleton item without wrapper"),
+ errhint("Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.")));
+ }
+
+ if (first)
+ return JsonbPGetDatum(JsonbValueToJsonb(first));
+
+ *empty = true;
+ return PointerGetDatum(NULL);
+}
+
+/* Executor-callable JSON_VALUE implementation */
+JsonbValue *
+JsonPathValue(Datum jb, JsonPath *jp, bool *empty, bool *error, List *vars)
+{
+ JsonbValue *res;
+ JsonValueList found = {0};
+ JsonPathExecResult jper PG_USED_FOR_ASSERTS_ONLY;
+ int count;
+
+ jper = executeJsonPath(jp, vars, GetJsonPathVar, DatumGetJsonbP(jb),
+ !error, &found, true);
+
+ Assert(error || !jperIsError(jper));
+
+ if (error && jperIsError(jper))
+ {
+ *error = true;
+ *empty = false;
+ return NULL;
+ }
+
+ count = JsonValueListLength(&found);
+
+ *empty = (count == 0);
+
+ if (*empty)
+ return NULL;
+
+ if (count > 1)
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_MORE_THAN_ONE_SQL_JSON_ITEM),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ res = JsonValueListHead(&found);
+
+ if (res->type == jbvBinary &&
+ JsonContainerIsScalar(res->val.binary.data))
+ JsonbExtractScalar(res->val.binary.data, res);
+
+ if (!IsAJsonbScalar(res))
+ {
+ if (error)
+ {
+ *error = true;
+ return NULL;
+ }
+
+ ereport(ERROR,
+ (errcode(ERRCODE_SQL_JSON_SCALAR_REQUIRED),
+ errmsg("JSON path expression in JSON_VALUE should return singleton scalar item")));
+ }
+
+ if (res->type == jbvNull)
+ return NULL;
+
+ return res;
+}
+
+static void
+JsonbValueInitNumericDatum(JsonbValue *jbv, Datum num)
+{
+ jbv->type = jbvNumeric;
+ jbv->val.numeric = DatumGetNumeric(num);
+}
+
+void
+JsonItemFromDatum(Datum val, Oid typid, int32 typmod, JsonbValue *res)
+{
+ switch (typid)
+ {
+ case BOOLOID:
+ res->type = jbvBool;
+ res->val.boolean = DatumGetBool(val);
+ break;
+ case NUMERICOID:
+ JsonbValueInitNumericDatum(res, val);
+ break;
+ case INT2OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int2_numeric, val));
+ break;
+ case INT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int4_numeric, val));
+ break;
+ case INT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(int8_numeric, val));
+ break;
+ case FLOAT4OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float4_numeric, val));
+ break;
+ case FLOAT8OID:
+ JsonbValueInitNumericDatum(res, DirectFunctionCall1(float8_numeric, val));
+ break;
+ case TEXTOID:
+ case VARCHAROID:
+ res->type = jbvString;
+ res->val.string.val = VARDATA_ANY(val);
+ res->val.string.len = VARSIZE_ANY_EXHDR(val);
+ break;
+ case DATEOID:
+ case TIMEOID:
+ case TIMETZOID:
+ case TIMESTAMPOID:
+ case TIMESTAMPTZOID:
+ res->type = jbvDatetime;
+ res->val.datetime.value = val;
+ res->val.datetime.typid = typid;
+ res->val.datetime.typmod = typmod;
+ res->val.datetime.tz = 0;
+ break;
+ case JSONBOID:
+ {
+ JsonbValue *jbv = res;
+ Jsonb *jb = DatumGetJsonbP(val);
+
+ if (JsonContainerIsScalar(&jb->root))
+ {
+ bool result PG_USED_FOR_ASSERTS_ONLY;
+
+ result = JsonbExtractScalar(&jb->root, jbv);
+ Assert(result);
+ }
+ else
+ JsonbInitBinary(jbv, jb);
+ break;
+ }
+ case JSONOID:
+ {
+ text *txt = DatumGetTextP(val);
+ char *str = text_to_cstring(txt);
+ Jsonb *jb;
+
+ jb = DatumGetJsonbP(DirectFunctionCall1(jsonb_in,
+ CStringGetDatum(str)));
+ pfree(str);
+
+ JsonItemFromDatum(JsonbPGetDatum(jb), JSONBOID, -1, res);
+ break;
+ }
+ default:
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+ errmsg("only datetime, bool, numeric, and text types can be casted to jsonpath types")));
+ }
+}
diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c
index ed7f40f053..342a2cf6aa 100644
--- a/src/backend/utils/adt/ruleutils.c
+++ b/src/backend/utils/adt/ruleutils.c
@@ -476,6 +476,8 @@ static void get_const_expr(Const *constval, deparse_context *context,
int showtype);
static void get_const_collation(Const *constval, deparse_context *context);
static void get_json_format(JsonFormat *format, StringInfo buf);
+static void get_json_returning(JsonReturning *returning, StringInfo buf,
+ bool json_format_by_default);
static void get_json_constructor(JsonConstructorExpr *ctor,
deparse_context *context, bool showimplicit);
static void get_json_constructor_options(JsonConstructorExpr *ctor,
@@ -518,6 +520,8 @@ static char *generate_qualified_type_name(Oid typid);
static text *string_to_text(char *str);
static char *flatten_reloptions(Oid relid);
static void get_reloptions(StringInfo buf, Datum reloptions);
+static void get_json_path_spec(Node *path_spec, deparse_context *context,
+ bool showimplicit);
#define only_marker(rte) ((rte)->inh ? "" : "ONLY ")
@@ -8300,6 +8304,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_WindowFunc:
case T_FuncExpr:
case T_JsonConstructorExpr:
+ case T_JsonExpr:
/* function-like: name(..) or name[..] */
return true;
@@ -8471,6 +8476,7 @@ isSimpleNode(Node *node, Node *parentNode, int prettyFlags)
case T_GroupingFunc: /* own parentheses */
case T_WindowFunc: /* own parentheses */
case T_CaseExpr: /* other separators */
+ case T_JsonExpr: /* own parentheses */
return true;
default:
return false;
@@ -8586,6 +8592,65 @@ get_rule_expr_paren(Node *node, deparse_context *context,
appendStringInfoChar(context->buf, ')');
}
+static void
+get_json_behavior(JsonBehavior *behavior, deparse_context *context,
+ const char *on)
+{
+ /*
+ * The order of array elements must correspond to the order of
+ * JsonBehaviorType members.
+ */
+ const char *behavior_names[] =
+ {
+ " NULL",
+ " ERROR",
+ " EMPTY",
+ " TRUE",
+ " FALSE",
+ " UNKNOWN",
+ " EMPTY ARRAY",
+ " EMPTY OBJECT",
+ " DEFAULT "
+ };
+
+ if ((int) behavior->btype < 0 || behavior->btype >= lengthof(behavior_names))
+ elog(ERROR, "invalid json behavior type: %d", behavior->btype);
+
+ appendStringInfoString(context->buf, behavior_names[behavior->btype]);
+
+ if (behavior->btype == JSON_BEHAVIOR_DEFAULT)
+ get_rule_expr(behavior->default_expr, context, false);
+
+ appendStringInfo(context->buf, " ON %s", on);
+}
+
+/*
+ * get_json_expr_options
+ *
+ * Parse back common options for JSON_QUERY, JSON_VALUE, JSON_EXISTS.
+ */
+static void
+get_json_expr_options(JsonExpr *jsexpr, deparse_context *context,
+ JsonBehaviorType default_behavior)
+{
+ if (jsexpr->op == JSON_QUERY_OP)
+ {
+ if (jsexpr->wrapper == JSW_CONDITIONAL)
+ appendStringInfo(context->buf, " WITH CONDITIONAL WRAPPER");
+ else if (jsexpr->wrapper == JSW_UNCONDITIONAL)
+ appendStringInfo(context->buf, " WITH UNCONDITIONAL WRAPPER");
+
+ if (jsexpr->omit_quotes)
+ appendStringInfo(context->buf, " OMIT QUOTES");
+ }
+
+ if (jsexpr->op != JSON_EXISTS_OP &&
+ jsexpr->on_empty->btype != default_behavior)
+ get_json_behavior(jsexpr->on_empty, context, "EMPTY");
+
+ if (jsexpr->on_error->btype != default_behavior)
+ get_json_behavior(jsexpr->on_error, context, "ERROR");
+}
/* ----------
* get_rule_expr - Parse back an expression
@@ -9745,6 +9810,7 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+
case T_JsonValueExpr:
{
JsonValueExpr *jve = (JsonValueExpr *) node;
@@ -9794,6 +9860,64 @@ get_rule_expr(Node *node, deparse_context *context,
}
break;
+ case T_JsonExpr:
+ {
+ JsonExpr *jexpr = (JsonExpr *) node;
+
+ switch (jexpr->op)
+ {
+ case JSON_EXISTS_OP:
+ appendStringInfoString(buf, "JSON_EXISTS(");
+ break;
+ case JSON_QUERY_OP:
+ appendStringInfoString(buf, "JSON_QUERY(");
+ break;
+ case JSON_VALUE_OP:
+ appendStringInfoString(buf, "JSON_VALUE(");
+ break;
+ }
+
+ get_rule_expr(jexpr->formatted_expr, context, showimplicit);
+
+ appendStringInfoString(buf, ", ");
+
+ get_json_path_spec(jexpr->path_spec, context, showimplicit);
+
+ if (jexpr->passing_values)
+ {
+ ListCell *lc1,
+ *lc2;
+ bool needcomma = false;
+
+ appendStringInfoString(buf, " PASSING ");
+
+ forboth(lc1, jexpr->passing_names,
+ lc2, jexpr->passing_values)
+ {
+ if (needcomma)
+ appendStringInfoString(buf, ", ");
+ needcomma = true;
+
+ get_rule_expr((Node *) lfirst(lc2), context, showimplicit);
+ appendStringInfo(buf, " AS %s",
+ ((String *) lfirst_node(String, lc1))->sval);
+ }
+ }
+
+ if (jexpr->op != JSON_EXISTS_OP ||
+ jexpr->returning->typid != BOOLOID)
+ get_json_returning(jexpr->returning, context->buf,
+ jexpr->op == JSON_QUERY_OP);
+
+ get_json_expr_options(jexpr, context,
+ jexpr->op != JSON_EXISTS_OP ?
+ JSON_BEHAVIOR_NULL :
+ JSON_BEHAVIOR_FALSE);
+
+ appendStringInfoString(buf, ")");
+ }
+ break;
+
case T_List:
{
char *sep;
@@ -9917,6 +10041,7 @@ looks_like_function(Node *node)
case T_MinMaxExpr:
case T_SQLValueFunction:
case T_XmlExpr:
+ case T_JsonExpr:
/* these are all accepted by func_expr_common_subexpr */
return true;
default:
@@ -10786,6 +10911,18 @@ get_const_collation(Const *constval, deparse_context *context)
}
}
+/*
+ * get_json_path_spec - Parse back a JSON path specification
+ */
+static void
+get_json_path_spec(Node *path_spec, deparse_context *context, bool showimplicit)
+{
+ if (IsA(path_spec, Const))
+ get_const_expr((Const *) path_spec, context, -1);
+ else
+ get_rule_expr(path_spec, context, showimplicit);
+}
+
/*
* get_json_format - Parse back a JsonFormat node
*/
diff --git a/src/include/executor/execExpr.h b/src/include/executor/execExpr.h
index c4fd933154..b1f1411ea9 100644
--- a/src/include/executor/execExpr.h
+++ b/src/include/executor/execExpr.h
@@ -23,6 +23,8 @@ struct ExprEvalStep;
struct SubscriptingRefState;
struct ScalarArrayOpExprHashTable;
struct JsonConstructorExprState;
+struct JsonbValue;
+struct JsonExprState;
/* Bits in ExprState->flags (see also execnodes.h for public flag bits): */
/* expression's interpreter has been initialized */
@@ -240,6 +242,9 @@ typedef enum ExprEvalOp
EEOP_XMLEXPR,
EEOP_JSON_CONSTRUCTOR,
EEOP_IS_JSON,
+ EEOP_JSONEXPR_PATH,
+ EEOP_JSONEXPR_COERCION,
+ EEOP_JSONEXPR_COERCION_FINISH,
EEOP_AGGREF,
EEOP_GROUPING_FUNC,
EEOP_WINDOW_FUNC,
@@ -692,6 +697,11 @@ typedef struct ExprEvalStep
JsonIsPredicate *pred; /* original expression node */
} is_json;
+ /* for EEOP_JSONEXPR_PATH */
+ struct
+ {
+ struct JsonExprState *jsestate;
+ } jsonexpr;
} d;
} ExprEvalStep;
@@ -755,6 +765,104 @@ typedef struct JsonConstructorExprState
int nargs;
} JsonConstructorExprState;
+/*
+ * Information about the state of JsonPath* evaluation.
+ */
+typedef struct JsonExprPostEvalState
+{
+ /* Did JsonPath* evaluation cause an error? */
+ NullableDatum error;
+
+ /* Is the result of JsonPath* evaluation empty? */
+ NullableDatum empty;
+
+ /*
+ * ExecEvalJsonExprPath() will set this to the address of the step to
+ * use to coerce the result of JsonPath* evaluation to the RETURNING
+ * type. Also see the description of possible step addresses this
+ * could be set to in the definition of JsonExprState.ZZ
+ */
+#define FIELDNO_JSONEXPRPOSTEVALSTATE_JUMP_EVAL_COERCION 2
+ int jump_eval_coercion;
+ bool throw_coercion_error;
+} JsonExprPostEvalState;
+
+/* State for evaluating a JsonExpr, too big to inline */
+typedef struct JsonExprState
+{
+ /* original expression node */
+ JsonExpr *jsexpr;
+
+ /* value/isnull for formatted_expr */
+ NullableDatum formatted_expr;
+
+ /* value/isnull for pathspec */
+ NullableDatum pathspec;
+
+ /* JsonPathVariable entries for passing_values */
+ List *args;
+
+ /*
+ * Per-row result status info populated by ExecEvalJsonExprPath(),
+ * ExecEvalJsonCoercionViaPopulateOrIO(), and
+ * ExecEvalJsonCoercionFinish().
+ */
+ JsonExprPostEvalState post_eval;
+
+ /*
+ * Address of the step that implements the non-ERROR variant of ON ERROR
+ * and ON EMPTY behaviors, to be jumped to when ExecEvalJsonExprPath()
+ * returns false on encountering an error during JsonPath* evaluation
+ * (ON ERROR) or on finding that no matching JSON item was returned (ON
+ * EMPTY). The same steps are also performed on encountering an error
+ * when coercing JsonPath* result to the RETURNING type.
+ */
+ int jump_error;
+
+ /*
+ * Addresses of steps to perform the coercion of the JsonPath* result value
+ * to the RETURNING type. Each address points to either 1) a special
+ * EEOP_JSONEXPR_COERCION step that handles coercion using the RETURNING
+ * type's input function or by using json_via_populate(), or 2) an
+ * expression such as CoerceViaIO. It may be -1 if no coercion is
+ * necessary.
+ *
+ * jump_eval_result_coercion points to the step to evaluate the coercion
+ * given in JsonExpr.result_coercion.
+ */
+ int jump_eval_result_coercion;
+
+ /* eval_item_coercion_jumps is an array of num_item_coercions elements
+ * each containing a step address to evaluate the coercion from a value of
+ * the given JsonItemType to the RETURNING type, or -1 if no coercion is
+ * necessary. item_coercion_via_expr is an array of boolean flags of the
+ * same length that indicates whether each valid step address in the
+ * eval_item_coercion_jumps array points to an expression or a
+ * EEOP_JSONEXPR_COERCION step. ExecEvalJsonExprPath() will cause an
+ * error if it's the latter, because that mode of coercion is not
+ * supported for all JsonItemTypes.
+ */
+ int num_item_coercions;
+ int *eval_item_coercion_jumps;
+ bool *item_coercion_via_expr;
+
+ /*
+ * For passing when initializing a EEOP_IOCOERCE_SAFE step for any
+ * CoerceViaIO nodes in the expression that must be evaluated in an
+ * error-safe manner.
+ */
+ ErrorSaveContext escontext;
+
+ /* Input function info for the RETURNING type. */
+ struct
+ {
+ FmgrInfo *finfo;
+ Oid typioparam;
+ } input;
+
+ /* Cache for json_populate_type() called for coercion in some cases */
+ void *cache;
+} JsonExprState;
/* functions in execExpr.c */
extern void ExprEvalPushStep(ExprState *es, const ExprEvalStep *s);
@@ -809,6 +917,11 @@ extern void ExecEvalXmlExpr(ExprState *state, ExprEvalStep *op);
extern void ExecEvalJsonConstructor(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
extern void ExecEvalJsonIsPredicate(ExprState *state, ExprEvalStep *op);
+extern void ExecEvalJsonCoercionViaPopulateOrIO(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
+extern void ExecEvalJsonCoercionFinish(ExprState *state, ExprEvalStep *op);
+extern bool ExecEvalJsonExprPath(ExprState *state, ExprEvalStep *op,
+ ExprContext *econtext);
extern void ExecEvalGroupingFunc(ExprState *state, ExprEvalStep *op);
extern void ExecEvalSubPlan(ExprState *state, ExprEvalStep *op,
ExprContext *econtext);
diff --git a/src/include/fmgr.h b/src/include/fmgr.h
index edf61e53f3..8a6b126de1 100644
--- a/src/include/fmgr.h
+++ b/src/include/fmgr.h
@@ -85,6 +85,7 @@ typedef struct FmgrInfo
typedef struct FunctionCallInfoBaseData
{
FmgrInfo *flinfo; /* ptr to lookup info used for this call */
+#define FIELDNO_FUNCTIONCALLINFODATA_CONTEXT 1
fmNodePtr context; /* pass info about context of call */
fmNodePtr resultinfo; /* pass or return extra info about result */
Oid fncollation; /* collation for function to use */
diff --git a/src/include/jit/llvmjit.h b/src/include/jit/llvmjit.h
index 3ab86de3ac..de9d8d36c5 100644
--- a/src/include/jit/llvmjit.h
+++ b/src/include/jit/llvmjit.h
@@ -85,6 +85,7 @@ extern PGDLLIMPORT LLVMTypeRef StructAggState;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerTransData;
extern PGDLLIMPORT LLVMTypeRef StructAggStatePerGroupData;
extern PGDLLIMPORT LLVMTypeRef StructPlanState;
+extern PGDLLIMPORT LLVMTypeRef StructJsonExprPostEvalState;
extern PGDLLIMPORT LLVMValueRef AttributeTemplate;
extern PGDLLIMPORT LLVMValueRef ExecEvalBoolSubroutineTemplate;
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 3180703005..783e82ac92 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -112,6 +112,10 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
int location);
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
+extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
+extern JsonBehaviorClause *makeJsonBehaviorClause(JsonBehavior *on_empty,
+ JsonBehavior *on_error,
+ int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index e494309da8..ce6893fb91 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1692,6 +1692,35 @@ typedef struct TriggerTransition
/* Nodes for SQL/JSON support */
+/*
+ * JsonQuotes -
+ * representation of [KEEP|OMIT] QUOTES clause for JSON_QUERY()
+ */
+typedef enum JsonQuotes
+{
+ JS_QUOTES_UNSPEC, /* unspecified */
+ JS_QUOTES_KEEP, /* KEEP QUOTES */
+ JS_QUOTES_OMIT /* OMIT QUOTES */
+} JsonQuotes;
+
+/*
+ * JsonBehaviorClause -
+ * representation of JSON ON ERROR / ON EMPTY clause
+ */
+typedef struct JsonBehaviorClause
+{
+ NodeTag type;
+ JsonBehavior *on_empty; /* behavior ON EMPTY */
+ JsonBehavior *on_error; /* behavior ON ERROR */
+ int location; /* token location, or -1 if unknown */
+} JsonBehaviorClause;
+
+/*
+ * JsonPathSpec -
+ * representation of JSON path constant
+ */
+typedef char *JsonPathSpec;
+
/*
* JsonOutput -
* representation of JSON output clause (RETURNING type [FORMAT format])
@@ -1703,6 +1732,47 @@ typedef struct JsonOutput
JsonReturning *returning; /* RETURNING FORMAT clause and type Oids */
} JsonOutput;
+/*
+ * JsonArgument -
+ * representation of argument from JSON PASSING clause
+ */
+typedef struct JsonArgument
+{
+ NodeTag type;
+ JsonValueExpr *val; /* argument value expression */
+ char *name; /* argument name */
+} JsonArgument;
+
+/*
+ * JsonCommon -
+ * representation of common syntax of functions using JSON path
+ */
+typedef struct JsonCommon
+{
+ NodeTag type;
+ JsonValueExpr *expr; /* context item expression */
+ Node *pathspec; /* JSON path specification expression */
+ char *pathname; /* path name, if any */
+ List *passing; /* list of PASSING clause arguments, if any */
+ int location; /* token location, or -1 if unknown */
+} JsonCommon;
+
+/*
+ * JsonFuncExpr -
+ * untransformed representation of JSON function expressions
+ */
+typedef struct JsonFuncExpr
+{
+ NodeTag type;
+ JsonExprOp op; /* expression type */
+ JsonCommon *common; /* common syntax */
+ JsonOutput *output; /* output clause, if specified */
+ JsonBehaviorClause *behavior; /* ON ERROR / EMPTY behavior, if specified */
+ JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
+ JsonQuotes quotes; /* omit or keep quotes? (JSON_QUERY only) */
+ int location; /* token location, or -1 if unknown */
+} JsonFuncExpr;
+
/*
* JsonKeyValue -
* untransformed representation of JSON object key-value pair for
diff --git a/src/include/nodes/primnodes.h b/src/include/nodes/primnodes.h
index bb930afb52..3f75c4608e 100644
--- a/src/include/nodes/primnodes.h
+++ b/src/include/nodes/primnodes.h
@@ -1552,6 +1552,17 @@ typedef struct XmlExpr
int location;
} XmlExpr;
+/*
+ * JsonExprOp -
+ * enumeration of JSON functions using JSON path
+ */
+typedef enum JsonExprOp
+{
+ JSON_VALUE_OP, /* JSON_VALUE() */
+ JSON_QUERY_OP, /* JSON_QUERY() */
+ JSON_EXISTS_OP /* JSON_EXISTS() */
+} JsonExprOp;
+
/*
* JsonEncoding -
* representation of JSON ENCODING clause
@@ -1576,6 +1587,37 @@ typedef enum JsonFormatType
* jsonb */
} JsonFormatType;
+/*
+ * JsonBehaviorType -
+ * enumeration of behavior types used in JSON ON ... BEHAVIOR clause
+ *
+ * If enum members are reordered, get_json_behavior() from ruleutils.c
+ * must be updated accordingly.
+ */
+typedef enum JsonBehaviorType
+{
+ JSON_BEHAVIOR_NULL = 0,
+ JSON_BEHAVIOR_ERROR,
+ JSON_BEHAVIOR_EMPTY,
+ JSON_BEHAVIOR_TRUE,
+ JSON_BEHAVIOR_FALSE,
+ JSON_BEHAVIOR_UNKNOWN,
+ JSON_BEHAVIOR_EMPTY_ARRAY,
+ JSON_BEHAVIOR_EMPTY_OBJECT,
+ JSON_BEHAVIOR_DEFAULT
+} JsonBehaviorType;
+
+/*
+ * JsonWrapper -
+ * representation of WRAPPER clause for JSON_QUERY()
+ */
+typedef enum JsonWrapper
+{
+ JSW_NONE,
+ JSW_CONDITIONAL,
+ JSW_UNCONDITIONAL,
+} JsonWrapper;
+
/*
* JsonFormat -
* representation of JSON FORMAT clause
@@ -1670,6 +1712,89 @@ typedef struct JsonIsPredicate
int location; /* token location, or -1 if unknown */
} JsonIsPredicate;
+/*
+ * JsonBehavior -
+ * representation of a given JSON behavior
+ */
+typedef struct JsonBehavior
+{
+ NodeTag type;
+ JsonBehaviorType btype; /* behavior type */
+ Node *default_expr; /* default expression when btype is
+ * JSON_BEHAVIOR_DEFAULT */
+ int location; /* token location, or -1 if unknown */
+} JsonBehavior;
+
+/*
+ * JsonItemType
+ * Represents type codes to identify a JsonCoercion node to use when
+ * coercing a given SQL/JSON items to the output SQL type
+ *
+ * The comment next to each item type mentions the JsonbValue.jbvType of the
+ * source JsonbValue value to be coerced using the expression in the
+ * JsonCoercion node.
+ *
+ * Also, see InitJsonItemCoercions() and ExecPrepareJsonItemCoercion().
+ */
+typedef enum JsonItemType
+{
+ JsonItemTypeNull = 0, /* jbvNull */
+ JsonItemTypeString = 1, /* jbvString */
+ JsonItemTypeNumeric = 2, /* jbvNumeric */
+ JsonItemTypeBoolean = 3, /* jbvBool */
+ JsonItemTypeDate = 4, /* jbvDatetime: DATEOID */
+ JsonItemTypeTime = 5, /* jbvDatetime: TIMEOID */
+ JsonItemTypeTimetz = 6, /* jbvDatetime: TIMETZOID */
+ JsonItemTypeTimestamp = 7, /* jbvDatetime: TIMESTAMPOID */
+ JsonItemTypeTimestamptz = 8, /* jbvDatetime: TIMESTAMPTZOID */
+ JsonItemTypeComposite = 9, /* jbvArray, jbvObject, jbvBinary */
+ JsonItemTypeInvalid = 10
+} JsonItemType;
+
+/*
+ * JsonCoercion -
+ * coercion from SQL/JSON item types to SQL types
+ */
+typedef struct JsonCoercion
+{
+ NodeTag type;
+ Node *expr; /* resulting expression coerced to target type */
+ bool via_populate; /* coerce result using json_populate_type()? */
+ bool via_io; /* coerce result using type input function? */
+ Oid collation; /* collation for coercion via I/O or populate */
+} JsonCoercion;
+
+typedef struct JsonItemCoercion
+{
+ NodeTag type;
+
+ JsonItemType item_type;
+ JsonCoercion *coercion;
+} JsonItemCoercion;
+
+/*
+ * JsonExpr -
+ * transformed representation of JSON_VALUE(), JSON_QUERY(), JSON_EXISTS()
+ */
+typedef struct JsonExpr
+{
+ Expr xpr;
+ JsonExprOp op; /* json function ID */
+ Node *formatted_expr; /* formatted context item expression */
+ JsonCoercion *result_coercion; /* resulting coercion to RETURNING type */
+ JsonFormat *format; /* context item format (JSON/JSONB) */
+ Node *path_spec; /* JSON path specification expression */
+ List *passing_names; /* PASSING argument names */
+ List *passing_values; /* PASSING argument values */
+ JsonReturning *returning; /* RETURNING clause type/format info */
+ JsonBehavior *on_empty; /* ON EMPTY behavior */
+ JsonBehavior *on_error; /* ON ERROR behavior */
+ List *item_coercions; /* coercions for JSON_VALUE */
+ JsonWrapper wrapper; /* WRAPPER for JSON_QUERY */
+ bool omit_quotes; /* KEEP/OMIT QUOTES for JSON_QUERY */
+ int location; /* token location, or -1 if unknown */
+} JsonExpr;
+
/* ----------------
* NullTest
*
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index 5984dcfa4b..0954d9fc7b 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -93,6 +93,7 @@ PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("conditional", CONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("connection", CONNECTION, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -147,11 +148,13 @@ PG_KEYWORD("double", DOUBLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("drop", DROP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("each", EACH, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("else", ELSE, RESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("empty", EMPTY_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enable", ENABLE_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encoding", ENCODING, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("encrypted", ENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("end", END_P, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("enum", ENUM_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("error", ERROR_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("escape", ESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("event", EVENT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("except", EXCEPT, RESERVED_KEYWORD, AS_LABEL)
@@ -233,10 +236,14 @@ PG_KEYWORD("join", JOIN, TYPE_FUNC_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json", JSON, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_array", JSON_ARRAY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_arrayagg", JSON_ARRAYAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_exists", JSON_EXISTS, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_object", JSON_OBJECT, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_query", JSON_QUERY, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_scalar", JSON_SCALAR, COL_NAME_KEYWORD, BARE_LABEL)
PG_KEYWORD("json_serialize", JSON_SERIALIZE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("json_value", JSON_VALUE, COL_NAME_KEYWORD, BARE_LABEL)
+PG_KEYWORD("keep", KEEP, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("keys", KEYS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -302,6 +309,7 @@ PG_KEYWORD("off", OFF, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("offset", OFFSET, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("oids", OIDS, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("old", OLD, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("omit", OMIT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("on", ON, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("only", ONLY, RESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("operator", OPERATOR, UNRESERVED_KEYWORD, BARE_LABEL)
@@ -344,6 +352,7 @@ PG_KEYWORD("procedures", PROCEDURES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("program", PROGRAM, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("publication", PUBLICATION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("quote", QUOTE, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("quotes", QUOTES, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("range", RANGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("read", READ, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("real", REAL, COL_NAME_KEYWORD, BARE_LABEL)
@@ -414,6 +423,7 @@ PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("string", STRING_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD, BARE_LABEL)
@@ -449,6 +459,7 @@ PG_KEYWORD("types", TYPES_P, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uescape", UESCAPE, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unbounded", UNBOUNDED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("uncommitted", UNCOMMITTED, UNRESERVED_KEYWORD, BARE_LABEL)
+PG_KEYWORD("unconditional", UNCONDITIONAL, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("unencrypted", UNENCRYPTED, UNRESERVED_KEYWORD, BARE_LABEL)
PG_KEYWORD("union", UNION, RESERVED_KEYWORD, AS_LABEL)
PG_KEYWORD("unique", UNIQUE, RESERVED_KEYWORD, BARE_LABEL)
diff --git a/src/include/utils/formatting.h b/src/include/utils/formatting.h
index 0cad3a2709..9a9ae9754e 100644
--- a/src/include/utils/formatting.h
+++ b/src/include/utils/formatting.h
@@ -29,5 +29,6 @@ extern char *asc_initcap(const char *buff, size_t nbytes);
extern Datum parse_datetime(text *date_txt, text *fmt, Oid collid, bool strict,
Oid *typid, int32 *typmod, int *tz,
struct Node *escontext);
+extern bool datetime_format_has_tz(const char *fmt_str);
#endif
diff --git a/src/include/utils/jsonb.h b/src/include/utils/jsonb.h
index addc9b608e..1da408a490 100644
--- a/src/include/utils/jsonb.h
+++ b/src/include/utils/jsonb.h
@@ -422,6 +422,9 @@ extern char *JsonbToCString(StringInfo out, JsonbContainer *in,
int estimated_len);
extern char *JsonbToCStringIndent(StringInfo out, JsonbContainer *in,
int estimated_len);
+extern Jsonb *JsonbMakeEmptyArray(void);
+extern Jsonb *JsonbMakeEmptyObject(void);
+extern char *JsonbUnquote(Jsonb *jb);
extern bool JsonbExtractScalar(JsonbContainer *jbc, JsonbValue *res);
extern const char *JsonbTypeName(JsonbValue *val);
diff --git a/src/include/utils/jsonfuncs.h b/src/include/utils/jsonfuncs.h
index 1c6d2be025..4c41eb5540 100644
--- a/src/include/utils/jsonfuncs.h
+++ b/src/include/utils/jsonfuncs.h
@@ -15,6 +15,7 @@
#define JSONFUNCS_H
#include "common/jsonapi.h"
+#include "nodes/nodes.h"
#include "utils/jsonb.h"
/*
@@ -87,5 +88,9 @@ extern Datum datum_to_json(Datum val, JsonTypeCategory tcategory,
extern Datum datum_to_jsonb(Datum val, JsonTypeCategory tcategory,
Oid outfuncoid);
extern Datum jsonb_from_text(text *js, bool unique_keys);
+extern Datum json_populate_type(Datum json_val, Oid json_type,
+ Oid typid, int32 typmod,
+ void **cache, MemoryContext mcxt, bool *isnull,
+ Node *escontext);
#endif
diff --git a/src/include/utils/jsonpath.h b/src/include/utils/jsonpath.h
index f0181e045f..5a37133847 100644
--- a/src/include/utils/jsonpath.h
+++ b/src/include/utils/jsonpath.h
@@ -16,7 +16,9 @@
#include "fmgr.h"
#include "nodes/pg_list.h"
+#include "nodes/primnodes.h"
#include "utils/jsonb.h"
+#include "utils/jsonfuncs.h"
typedef struct
{
@@ -184,6 +186,7 @@ extern bool jspGetBool(JsonPathItem *v);
extern char *jspGetString(JsonPathItem *v, int32 *len);
extern bool jspGetArraySubscript(JsonPathItem *v, JsonPathItem *from,
JsonPathItem *to, int i);
+extern bool jspIsMutable(JsonPath *path, List *varnames, List *varexprs);
extern const char *jspOperationName(JsonPathItemType type);
@@ -261,4 +264,28 @@ extern bool jspConvertRegexFlags(uint32 xflags, int *result,
struct Node *escontext);
+/*
+ * Evaluation of jsonpath
+ */
+
+/* External variable passed into jsonpath. */
+typedef struct JsonPathVariable
+{
+ char *name;
+ Oid typid;
+ int32 typmod;
+ Datum value;
+ bool isnull;
+} JsonPathVariable;
+
+/* SQL/JSON item */
+extern void JsonItemFromDatum(Datum val, Oid typid, int32 typmod,
+ JsonbValue *res);
+
+extern bool JsonPathExists(Datum jb, JsonPath *path, bool *error, List *vars);
+extern Datum JsonPathQuery(Datum jb, JsonPath *jp, JsonWrapper wrapper,
+ bool *empty, bool *error, List *vars);
+extern JsonbValue *JsonPathValue(Datum jb, JsonPath *jp, bool *empty,
+ bool *error, List *vars);
+
#endif
diff --git a/src/interfaces/ecpg/preproc/ecpg.trailer b/src/interfaces/ecpg/preproc/ecpg.trailer
index 435c139ec2..b2aa44f36d 100644
--- a/src/interfaces/ecpg/preproc/ecpg.trailer
+++ b/src/interfaces/ecpg/preproc/ecpg.trailer
@@ -651,6 +651,34 @@ var_type: simple_type
$$.type_index = mm_strdup("-1");
$$.type_sizeof = NULL;
}
+ | STRING_P
+ {
+ if (INFORMIX_MODE)
+ {
+ /* In Informix mode, "string" is automatically a typedef */
+ $$.type_enum = ECPGt_string;
+ $$.type_str = mm_strdup("char");
+ $$.type_dimension = mm_strdup("-1");
+ $$.type_index = mm_strdup("-1");
+ $$.type_sizeof = NULL;
+ }
+ else
+ {
+ /* Otherwise, legal only if user typedef'ed it */
+ struct typedefs *this = get_typedef("string", false);
+
+ $$.type_str = (this->type->type_enum == ECPGt_varchar || this->type->type_enum == ECPGt_bytea) ? EMPTY : mm_strdup(this->name);
+ $$.type_enum = this->type->type_enum;
+ $$.type_dimension = this->type->type_dimension;
+ $$.type_index = this->type->type_index;
+ if (this->type->type_sizeof && strlen(this->type->type_sizeof) != 0)
+ $$.type_sizeof = this->type->type_sizeof;
+ else
+ $$.type_sizeof = cat_str(3, mm_strdup("sizeof("), mm_strdup(this->name), mm_strdup(")"));
+
+ struct_member_list[struct_level] = ECPGstruct_member_dup(this->struct_member_list);
+ }
+ }
| INTERVAL ecpg_interval
{
$$.type_enum = ECPGt_interval;
diff --git a/src/test/regress/expected/json_sqljson.out b/src/test/regress/expected/json_sqljson.out
new file mode 100644
index 0000000000..8b87580752
--- /dev/null
+++ b/src/test/regress/expected/json_sqljson.out
@@ -0,0 +1,18 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ERROR: JSON_EXISTS() is not yet implemented for the json type
+LINE 1: SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ERROR: JSON_VALUE() is not yet implemented for the json type
+LINE 1: SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
+-- JSON_QUERY
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ERROR: JSON_QUERY() is not yet implemented for the json type
+LINE 1: SELECT JSON_QUERY(NULL FORMAT JSON, '$');
+ ^
+HINT: Try casting the argument to jsonb
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
new file mode 100644
index 0000000000..b467971ed9
--- /dev/null
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -0,0 +1,1097 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+ json_exists
+-------------
+
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb 'null', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+ json_exists
+-------------
+ f
+(1 row)
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+ json_exists
+-------------
+ t
+(1 row)
+
+-- extension: RETURNING clause
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING bool);
+ json_exists
+-------------
+ t
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING bool);
+ json_exists
+-------------
+ f
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING int);
+ json_exists
+-------------
+ 1
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING int);
+ json_exists
+-------------
+ 0
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING text);
+ json_exists
+-------------
+ true
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING text);
+ json_exists
+-------------
+ false
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', 'strict $[1]' RETURNING text FALSE ON ERROR);
+ json_exists
+-------------
+ false
+(1 row)
+
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+ERROR: cannot cast type boolean to jsonb
+LINE 1: SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+ ^
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+ERROR: cannot cast type boolean to real
+LINE 1: SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+ ^
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSON); -- RETURNING FORMAT not allowed
+ERROR: cannot specify FORMAT in RETURNING clause of JSON_EXISTS()
+LINE 1: ...CT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSO...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING record);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+-- JSON_VALUE
+SELECT JSON_VALUE(NULL::jsonb, '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+ json_value
+------------
+ true
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+ json_value
+------------
+ t
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$');
+ json_value
+------------
+ 123
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+ json_value
+------------
+ 123
+(1 row)
+
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+ERROR: SQL/JSON item cannot be cast to target type
+SELECT JSON_VALUE(jsonb '1.23', '$');
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+ json_value
+------------
+ 1.23
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "1.23"
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+ json_value
+------------
+ aaa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+ json_value
+------------
+ aa
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+ json_value
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+ json_value
+------------
+ "\"aaa\""
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "aaa"
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+ json_value
+------------
+ 111
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+ ?column?
+----------
+ 357
+(1 row)
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+ ?column?
+------------
+ 03-01-2017
+(1 row)
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON EMPTY ERROR ON ERROR);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb ERROR ON ERROR);
+ERROR: value for domain rgb violates check constraint "rgb_check"
+SELECT JSON_VALUE(jsonb '[]', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '{}', '$');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '1', '$.a');
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+ERROR: jsonpath member accessor can only be applied to an object
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+ json_value
+------------
+ error
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+ json_value
+------------
+
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+ json_value
+------------
+ 2
+(1 row)
+
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+ json_value
+------------
+ 3
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_VALUE should return singleton scalar item
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+ json_value
+------------
+ 0
+(1 row)
+
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: " "
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 5
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+ json_value
+------------
+ 1
+(1 row)
+
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+ERROR: cannot specify FORMAT in RETURNING clause of JSON_VALUE()
+LINE 1: ...CT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSO...
+ ^
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+ x | y
+---+----
+ 0 | -2
+ 1 | 2
+ 2 | -1
+(3 rows)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point ERROR ON ERROR);
+ json_value
+------------
+ (1,2)
+(1 row)
+
+-- Test timestamptz passing and output
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+ json_value
+------------------------------
+ Tue Feb 20 18:34:56 2018 PST
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+ json_value
+--------------------------
+ Tue Feb 20 18:34:56 2018
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_value
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- JSON_QUERY
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+ json_query | json_query | json_query | json_query | json_query
+--------------------+--------------------+--------------------+----------------------+----------------------
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, null, "2"] | [1, null, "2"] | [1, null, "2"] | [[1, null, "2"]] | [[1, null, "2"]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+(6 rows)
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+ unspec | without | with cond | with uncond | with
+--------------------+--------------------+---------------------+----------------------+----------------------
+ | | | |
+ | | | |
+ null | null | [null] | [null] | [null]
+ 12.3 | 12.3 | [12.3] | [12.3] | [12.3]
+ true | true | [true] | [true] | [true]
+ "aaa" | "aaa" | ["aaa"] | ["aaa"] | ["aaa"]
+ [1, 2, 3] | [1, 2, 3] | [1, 2, 3] | [[1, 2, 3]] | [[1, 2, 3]]
+ {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | {"a": 1, "b": [2]} | [{"a": 1, "b": [2]}] | [{"a": 1, "b": [2]}]
+ | | [1, "2", null, [3]] | [1, "2", null, [3]] | [1, "2", null, [3]]
+(9 rows)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+ json_query
+------------
+ "aaa"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+ json_query
+------------
+ aaa
+(1 row)
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+ERROR: invalid input syntax for type json
+DETAIL: Token "aaa" is invalid.
+CONTEXT: JSON data, line 1: aaa
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+ json_query
+------------
+ \x616161
+(1 row)
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES)...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTE...
+ ^
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
+LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTE...
+ ^
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+ json_query
+------------
+ [1]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+ json_query
+------------
+ []
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+ERROR: no SQL/JSON item
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+ERROR: JSON path expression in JSON_QUERY should return singleton item without wrapper
+HINT: Use WITH WRAPPER clause to wrap SQL/JSON item sequence into array.
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+ json_query
+------------
+ "empty"
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+ json_query
+------------
+ [1,
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+ json_query
+------------
+ [1, 2]
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+ json_query
+----------------
+ \x5b312c20325d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ \x7b7d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ \x7b7d
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+ json_query
+------------
+ {}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+ERROR: expected JSON array
+SELECT JSON_QUERY(jsonb '"[3,4]"', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+ERROR: expected JSON array
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+ERROR: returning pseudo-types is not supported in SQL/JSON functions
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+ x | y | list
+---+---+--------------
+ 0 | 0 | []
+ 0 | 1 | [1]
+ 0 | 2 | [1, 2]
+ 0 | 3 | [1, 2, 3]
+ 0 | 4 | [1, 2, 3, 4]
+ 1 | 0 | []
+ 1 | 1 | [1]
+ 1 | 2 | [1, 2]
+ 1 | 3 | [1, 2, 3]
+ 1 | 4 | [1, 2, 3, 4]
+ 2 | 0 | []
+ 2 | 1 | []
+ 2 | 2 | [2]
+ 2 | 3 | [2, 3]
+ 2 | 4 | [2, 3, 4]
+ 3 | 0 | []
+ 3 | 1 | []
+ 3 | 2 | []
+ 3 | 3 | [3]
+ 3 | 4 | [3, 4]
+ 4 | 0 | []
+ 4 | 1 | []
+ 4 | 2 | []
+ 4 | 3 | []
+ 4 | 4 | [4]
+(25 rows)
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+-----------------------------------------------------
+ (1,aaa,"[1, ""2"", {}]","{""x"": [1, ""2"", {}]}",)
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+ unnest
+------------------------
+ {"a": 1, "b": ["foo"]}
+ {"a": 2, "c": {}}
+ 123
+(3 rows)
+
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath);
+ json_query
+------------
+
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath ERROR ON ERROR);
+ERROR: syntax error at or near "{" of jsonpath input
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+--------------
+ {1,2,NULL,3}
+(1 row)
+
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+ERROR: invalid input syntax for type integer: "a"
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+ json_query
+------------
+
+(1 row)
+
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+ a | t | js | jb | jsa
+---+-------------+----+------------+-----
+ 1 | ["foo", []] | | |
+ 2 | | | [{}, true] |
+(2 rows)
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+ json_query
+------------
+ 1
+(1 row)
+
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+ERROR: domain sqljsonb_int_not_null does not allow null values
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+ json_query
+-----------------------------
+ "2018-02-21T02:34:56+00:00"
+(1 row)
+
+-- Test constraints
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT '12' ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+ CONSTRAINT test_jsonb_constraint6
+ CHECK (JSON_EXISTS(js::jsonb, 'strict $.a' RETURNING int TRUE ON ERROR) < 2)
+);
+\d test_jsonb_constraints
+ Table "public.test_jsonb_constraints"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+--------------------------------------------------------------------------------
+ js | text | | |
+ i | integer | | |
+ x | jsonb | | | JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+Check constraints:
+ "test_jsonb_constraint1" CHECK (js IS JSON)
+ "test_jsonb_constraint2" CHECK (JSON_EXISTS(js::jsonb, '$."a"' PASSING i + 5 AS int, i::text AS txt, ARRAY[1, 2, 3] AS arr))
+ "test_jsonb_constraint3" CHECK (JSON_VALUE(js::jsonb, '$."a"' RETURNING integer DEFAULT 12 ON EMPTY ERROR ON ERROR) > i)
+ "test_jsonb_constraint4" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ "test_jsonb_constraint5" CHECK (JSON_QUERY(js::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+ "test_jsonb_constraint6" CHECK (JSON_EXISTS(js::jsonb, 'strict $."a"' RETURNING integer TRUE ON ERROR) < 2)
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+ check_clause
+------------------------------------------------------------------------------------------------------------------------
+ (JSON_EXISTS((js)::jsonb, 'strict $."a"' RETURNING integer TRUE ON ERROR) < 2)
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING character(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > ('a'::bpchar COLLATE "C"))
+ (JSON_QUERY((js)::jsonb, '$."a"' RETURNING jsonb WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < '[10]'::jsonb)
+ (JSON_VALUE((js)::jsonb, '$."a"' RETURNING integer DEFAULT 12 ON EMPTY ERROR ON ERROR) > i)
+ (js IS JSON)
+ JSON_EXISTS((js)::jsonb, '$."a"' PASSING (i + 5) AS int, (i)::text AS txt, ARRAY[1, 2, 3] AS arr)
+(6 rows)
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+ pg_get_expr
+--------------------------------------------------------------------------------
+ JSON_QUERY('[1, 2]'::jsonb, '$[*]' RETURNING jsonb WITH UNCONDITIONAL WRAPPER)
+(1 row)
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint1"
+DETAIL: Failing row contains (, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains (1, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ([], null, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint2"
+DETAIL: Failing row contains ({"b": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint3"
+DETAIL: Failing row contains ({"a": 1}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint5"
+DETAIL: Failing row contains ({"a": 7}, 1, [1, 2]).
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+ERROR: new row for relation "test_jsonb_constraints" violates check constraint "test_jsonb_constraint4"
+DETAIL: Failing row contains ({"a": 10}, 1, [1, 2]).
+DROP TABLE test_jsonb_constraints;
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+ERROR: functions in index expression must be marked IMMUTABLE
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index f0987ff537..864bf04fe7 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -103,7 +103,7 @@ test: select_views portals_p2 foreign_key cluster dependency guc bitmapops combo
# ----------
# Another group of parallel tests (JSON related)
# ----------
-test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson
+test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath sqljson json_sqljson jsonb_sqljson
# ----------
# Another group of parallel tests
diff --git a/src/test/regress/sql/json_sqljson.sql b/src/test/regress/sql/json_sqljson.sql
new file mode 100644
index 0000000000..4f30fa46b9
--- /dev/null
+++ b/src/test/regress/sql/json_sqljson.sql
@@ -0,0 +1,11 @@
+-- JSON_EXISTS
+
+SELECT JSON_EXISTS(NULL FORMAT JSON, '$');
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL FORMAT JSON, '$');
+
+-- JSON_QUERY
+
+SELECT JSON_QUERY(NULL FORMAT JSON, '$');
diff --git a/src/test/regress/sql/jsonb_sqljson.sql b/src/test/regress/sql/jsonb_sqljson.sql
new file mode 100644
index 0000000000..b3a32c09f6
--- /dev/null
+++ b/src/test/regress/sql/jsonb_sqljson.sql
@@ -0,0 +1,349 @@
+-- JSON_EXISTS
+SELECT JSON_EXISTS(NULL::jsonb, '$');
+
+SELECT JSON_EXISTS(jsonb '[]', '$');
+SELECT JSON_EXISTS(JSON_OBJECT(RETURNING jsonb), '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$');
+SELECT JSON_EXISTS(jsonb 'null', '$');
+SELECT JSON_EXISTS(jsonb '[]', '$');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a');
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_EXISTS(jsonb 'null', '$.a');
+SELECT JSON_EXISTS(jsonb '[]', '$.a');
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'strict $.a'); -- FALSE on error
+SELECT JSON_EXISTS(jsonb '[1, "aaa", {"a": 1}]', 'lax $.a');
+SELECT JSON_EXISTS(jsonb '{}', '$.a');
+SELECT JSON_EXISTS(jsonb '{"b": 1, "a": 2}', '$.a');
+
+SELECT JSON_EXISTS(jsonb '1', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": {"b": 1}}', '$.a.b');
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.a.b');
+
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING 1 AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x)' PASSING '1' AS x);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 2 AS y);
+SELECT JSON_EXISTS(jsonb '{"a": 1, "b": 2}', '$.* ? (@ > $x && @ < $y)' PASSING 0 AS x, 1 AS y);
+
+-- extension: boolean expressions
+SELECT JSON_EXISTS(jsonb '1', '$ > 2');
+SELECT JSON_EXISTS(jsonb '1', '$.a > 2' ERROR ON ERROR);
+
+-- extension: RETURNING clause
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING bool);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING bool);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING int);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING int);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING text);
+SELECT JSON_EXISTS(jsonb '1', '$[1]' RETURNING text);
+SELECT JSON_EXISTS(jsonb '1', 'strict $[1]' RETURNING text FALSE ON ERROR);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING jsonb);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4);
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING float4 FORMAT JSON); -- RETURNING FORMAT not allowed
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_EXISTS(jsonb '1', '$[0]' RETURNING record);
+
+
+-- JSON_VALUE
+
+SELECT JSON_VALUE(NULL::jsonb, '$');
+
+SELECT JSON_VALUE(jsonb 'null', '$');
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING int);
+
+SELECT JSON_VALUE(jsonb 'true', '$');
+SELECT JSON_VALUE(jsonb 'true', '$' RETURNING bool);
+
+SELECT JSON_VALUE(jsonb '123', '$');
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING int) + 234;
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING text);
+/* jsonb bytea ??? */
+SELECT JSON_VALUE(jsonb '123', '$' RETURNING bytea ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1.23', '$');
+SELECT JSON_VALUE(jsonb '1.23', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING numeric);
+SELECT JSON_VALUE(jsonb '"1.23"', '$' RETURNING int ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '"aaa"', '$');
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(5));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING char(2));
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING json ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING jsonb ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING json);
+SELECT JSON_VALUE(jsonb '"\"aaa\""', '$' RETURNING jsonb);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '"aaa"', '$' RETURNING int DEFAULT 111 ON ERROR);
+SELECT JSON_VALUE(jsonb '"123"', '$' RETURNING int) + 234;
+
+SELECT JSON_VALUE(jsonb '"2017-02-20"', '$' RETURNING date) + 9;
+
+-- Test NULL checks execution in domain types
+CREATE DOMAIN sqljsonb_int_not_null AS int NOT NULL;
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null);
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb 'null', '$' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT 2 ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', '$.a' RETURNING sqljsonb_int_not_null DEFAULT NULL ON EMPTY ERROR ON ERROR);
+CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple');
+CREATE DOMAIN rgb AS rainbow CHECK (VALUE IN ('red', 'green', 'blue'));
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb);
+SELECT JSON_VALUE('"purple"'::jsonb, 'lax $[*]' RETURNING rgb ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '[]', '$');
+SELECT JSON_VALUE(jsonb '[]', '$' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '{}', '$');
+SELECT JSON_VALUE(jsonb '{}', '$' ERROR ON ERROR);
+
+SELECT JSON_VALUE(jsonb '1', '$.a');
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 'error' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'strict $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT 2 ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' NULL ON EMPTY DEFAULT '2' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' DEFAULT '2' ON EMPTY DEFAULT '3' ON ERROR);
+SELECT JSON_VALUE(jsonb '1', 'lax $.a' ERROR ON EMPTY DEFAULT '3' ON ERROR);
+
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[1,2]', '$[*]' DEFAULT '0' ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int ERROR ON ERROR);
+SELECT JSON_VALUE(jsonb '[" "]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int DEFAULT 2 + 3 ON ERROR);
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING int FORMAT JSON); -- RETURNING FORMAT not allowed
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_VALUE(jsonb '["1"]', '$[*]' RETURNING record);
+
+SELECT
+ x,
+ JSON_VALUE(
+ jsonb '{"a": 1, "b": 2}',
+ '$.* ? (@ > $x)' PASSING x AS x
+ RETURNING int
+ DEFAULT -1 ON EMPTY
+ DEFAULT -2 ON ERROR
+ ) y
+FROM
+ generate_series(0, 2) x;
+
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a);
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point);
+SELECT JSON_VALUE(jsonb 'null', '$a' PASSING point ' (1, 2 )' AS a RETURNING point ERROR ON ERROR);
+
+-- Test timestamptz passing and output
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamptz);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING timestamp);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_VALUE(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- JSON_QUERY
+
+SELECT
+ JSON_QUERY(js, '$'),
+ JSON_QUERY(js, '$' WITHOUT WRAPPER),
+ JSON_QUERY(js, '$' WITH CONDITIONAL WRAPPER),
+ JSON_QUERY(js, '$' WITH UNCONDITIONAL ARRAY WRAPPER),
+ JSON_QUERY(js, '$' WITH ARRAY WRAPPER)
+FROM
+ (VALUES
+ (jsonb 'null'),
+ ('12.3'),
+ ('true'),
+ ('"aaa"'),
+ ('[1, null, "2"]'),
+ ('{"a": 1, "b": [2]}')
+ ) foo(js);
+
+SELECT
+ JSON_QUERY(js, 'strict $[*]') AS "unspec",
+ JSON_QUERY(js, 'strict $[*]' WITHOUT WRAPPER) AS "without",
+ JSON_QUERY(js, 'strict $[*]' WITH CONDITIONAL WRAPPER) AS "with cond",
+ JSON_QUERY(js, 'strict $[*]' WITH UNCONDITIONAL ARRAY WRAPPER) AS "with uncond",
+ JSON_QUERY(js, 'strict $[*]' WITH ARRAY WRAPPER) AS "with"
+FROM
+ (VALUES
+ (jsonb '1'),
+ ('[]'),
+ ('[null]'),
+ ('[12.3]'),
+ ('[true]'),
+ ('["aaa"]'),
+ ('[[1, 2, 3]]'),
+ ('[{"a": 1, "b": [2]}]'),
+ ('[1, "2", null, [3]]')
+ ) foo(js);
+
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text KEEP QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING text OMIT QUOTES ON SCALAR STRING);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING json OMIT QUOTES ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERROR ON ERROR);
+
+-- QUOTES behavior should not be specified when WITH WRAPPER used:
+-- Should fail
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
+-- Should succeed
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
+SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER KEEP QUOTES);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]');
+SELECT JSON_QUERY(jsonb '[]', '$[*]' NULL ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY ARRAY ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' EMPTY OBJECT ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' DEFAULT '"empty"' ON EMPTY);
+
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY NULL ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY ARRAY ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON EMPTY ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[]', '$[*]' ERROR ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' DEFAULT '"empty"' ON ERROR);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING json FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING jsonb FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(10));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING char(3));
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING text FORMAT JSON);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea);
+SELECT JSON_QUERY(jsonb '[1,2]', '$' RETURNING bytea FORMAT JSON);
+
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING bytea FORMAT JSON EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING json EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2]', '$[*]' RETURNING jsonb EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+SELECT JSON_QUERY(jsonb '"[3,4]"', '$[*]' RETURNING bigint[] EMPTY OBJECT ON ERROR);
+
+-- RETUGNING pseudo-types not allowed
+SELECT JSON_QUERY(jsonb '[3,4]', '$[*]' RETURNING anyarray EMPTY OBJECT ON ERROR);
+
+SELECT
+ x, y,
+ JSON_QUERY(
+ jsonb '[1,2,3,4,5,null]',
+ '$[*] ? (@ >= $x && @ <= $y)'
+ PASSING x AS x, y AS y
+ WITH CONDITIONAL WRAPPER
+ EMPTY ARRAY ON EMPTY
+ ) list
+FROM
+ generate_series(0, 4) x,
+ generate_series(0, 4) y;
+
+-- Extension: record types returning
+CREATE TYPE sqljsonb_rec AS (a int, t text, js json, jb jsonb, jsa json[]);
+CREATE TYPE sqljsonb_reca AS (reca sqljsonb_rec[]);
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[{"a": "a", "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING sqljsonb_rec);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"jsa": [{"a": 1, "b": ["foo"]}, {"a": 2, "c": {}}, 123]}', '$' RETURNING sqljsonb_rec)).jsa);
+SELECT * FROM unnest((JSON_QUERY(jsonb '{"reca": [{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]}', '$' RETURNING sqljsonb_reca)).reca);
+
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath);
+SELECT JSON_QUERY(jsonb '[{"a": 1, "b": "foo", "t": "aaa", "js": [1, "2", {}], "jb": {"x": [1, "2", {}]}}, {"a": 2}]', '$[0]' RETURNING jsonpath ERROR ON ERROR);
+
+-- Extension: array types returning
+SELECT JSON_QUERY(jsonb '[1,2,null,"3"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER ERROR ON ERROR);
+SELECT JSON_QUERY(jsonb '[1,2,null,"a"]', '$[*]' RETURNING int[] WITH WRAPPER);
+SELECT * FROM unnest(JSON_QUERY(jsonb '[{"a": 1, "t": ["foo", []]}, {"a": 2, "jb": [{}, true]}]', '$' RETURNING sqljsonb_rec[]));
+
+-- Extension: domain types returning
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.a' RETURNING sqljsonb_int_not_null);
+SELECT JSON_QUERY(jsonb '{"a": 1}', '$.b' RETURNING sqljsonb_int_not_null);
+
+-- Test timestamptz passing and output
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING json);
+SELECT JSON_QUERY(jsonb 'null', '$ts' PASSING timestamptz '2018-02-21 12:34:56 +10' AS ts RETURNING jsonb);
+
+-- Test constraints
+
+CREATE TABLE test_jsonb_constraints (
+ js text,
+ i int,
+ x jsonb DEFAULT JSON_QUERY(jsonb '[1,2]', '$[*]' WITH WRAPPER)
+ CONSTRAINT test_jsonb_constraint1
+ CHECK (js IS JSON)
+ CONSTRAINT test_jsonb_constraint2
+ CHECK (JSON_EXISTS(js::jsonb, '$.a' PASSING i + 5 AS int, i::text AS txt, array[1,2,3] as arr))
+ CONSTRAINT test_jsonb_constraint3
+ CHECK (JSON_VALUE(js::jsonb, '$.a' RETURNING int DEFAULT '12' ON EMPTY ERROR ON ERROR) > i)
+ CONSTRAINT test_jsonb_constraint4
+ CHECK (JSON_QUERY(js::jsonb, '$.a' WITH CONDITIONAL WRAPPER EMPTY OBJECT ON ERROR) < jsonb '[10]')
+ CONSTRAINT test_jsonb_constraint5
+ CHECK (JSON_QUERY(js::jsonb, '$.a' RETURNING char(5) OMIT QUOTES EMPTY ARRAY ON EMPTY) > 'a' COLLATE "C")
+ CONSTRAINT test_jsonb_constraint6
+ CHECK (JSON_EXISTS(js::jsonb, 'strict $.a' RETURNING int TRUE ON ERROR) < 2)
+);
+
+\d test_jsonb_constraints
+
+SELECT check_clause
+FROM information_schema.check_constraints
+WHERE constraint_name LIKE 'test_jsonb_constraint%'
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid)
+FROM pg_attrdef
+WHERE adrelid = 'test_jsonb_constraints'::regclass
+ORDER BY 1;
+
+SELECT pg_get_expr(adbin, adrelid) FROM pg_attrdef WHERE adrelid = 'test_jsonb_constraints'::regclass;
+
+INSERT INTO test_jsonb_constraints VALUES ('', 1);
+INSERT INTO test_jsonb_constraints VALUES ('1', 1);
+INSERT INTO test_jsonb_constraints VALUES ('[]');
+INSERT INTO test_jsonb_constraints VALUES ('{"b": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 1}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 7}', 1);
+INSERT INTO test_jsonb_constraints VALUES ('{"a": 10}', 1);
+
+DROP TABLE test_jsonb_constraints;
+
+-- Test mutabilily od query functions
+CREATE TABLE test_jsonb_mutability(js jsonb);
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a[0]'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime()'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@ < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime())'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime() < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("HH:MI TZH"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.a ? (@.datetime("HH:MI TZH") < $.datetime("YY-MM-DD HH:MI"))'));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("HH:MI TZH") < $y' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '12:34'::timetz AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() < $x' PASSING '1234'::int AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime() ? (@ == $x)' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$.datetime("YY-MM-DD") ? (@ == $x)' PASSING '2020-07-14'::date AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, 0 to $.a ? (@.datetime() == $x)]' PASSING '12:34'::time AS x));
+CREATE INDEX ON test_jsonb_mutability (JSON_QUERY(js, '$[1, $.a ? (@.datetime("HH:MI") == $x)]' PASSING '12:34'::time AS x));
+DROP TABLE test_jsonb_mutability;
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 92c0003ab1..5ce706205c 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1242,6 +1242,7 @@ Join
JoinCostWorkspace
JoinDomain
JoinExpr
+JsonFuncExpr
JoinHashEntry
JoinPath
JoinPathExtraData
@@ -1252,18 +1253,30 @@ JsObject
JsValue
JsonAggConstructor
JsonAggState
+JsonArgument
JsonArrayAgg
JsonArrayConstructor
JsonArrayQueryConstructor
JsonBaseObjectInfo
+JsonBehavior
+JsonBehaviorClause
+JsonBehaviorType
+JsonCoercion
+JsonCoercionState
+JsonCommon
JsonConstructorExpr
JsonConstructorExprState
JsonConstructorType
JsonEncoding
+JsonExpr
+JsonExprOp
+JsonExprPostEvalState
+JsonExprState
JsonFormat
JsonFormatType
JsonHashEntry
JsonIsPredicate
+JsonItemType
JsonIterateStringValuesAction
JsonKeyValue
JsonLexContext
@@ -1281,6 +1294,7 @@ JsonParseContext
JsonParseErrorType
JsonPath
JsonPathBool
+JsonPathDatatypeStatus
JsonPathExecContext
JsonPathExecResult
JsonPathGinAddPathItemFunc
@@ -1293,10 +1307,15 @@ JsonPathGinPathItem
JsonPathItem
JsonPathItemType
JsonPathKeyword
+JsonPathMutableContext
JsonPathParseItem
JsonPathParseResult
JsonPathPredicateCallback
JsonPathString
+JsonPathVarCallback
+JsonPathVariable
+JsonPathVariableEvalContext
+JsonQuotes
JsonReturning
JsonScalarExpr
JsonSemAction
@@ -1313,6 +1332,7 @@ JsonValueExpr
JsonValueList
JsonValueListIterator
JsonValueType
+JsonWrapper
Jsonb
JsonbAggState
JsonbContainer
--
2.35.3
^ permalink raw reply [nested|flat] 94+ messages in thread
* Re: remaining sql/json patches
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-16 04:53 ` Re: remaining sql/json patches Erik Rijkers <[email protected]>
2023-11-16 04:57 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-16 06:53 ` Re: remaining sql/json patches Amit Langote <[email protected]>
@ 2023-11-21 07:09 ` Peter Eisentraut <[email protected]>
0 siblings, 0 replies; 94+ messages in thread
From: Peter Eisentraut @ 2023-11-21 07:09 UTC (permalink / raw)
To: Amit Langote <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andres Freund <[email protected]>; Andrew Dunstan <[email protected]>; pgsql-hackers; jian he <[email protected]>; Erik Rijkers <[email protected]>
I looked a bit at the parser additions, because there were some concerns
expressed that they are quite big.
It looks like the parser rules were mostly literally copied from the BNF
in the SQL standard. That's probably a reasonable place to start, but
now at the end, there is some room for simplification.
Attached are a few patches that apply on top of the 0003 patch. (I
haven't gotten to 0004 in detail yet.) Some explanations:
0001-Put-keywords-in-right-order.patch
This is just an unrelated cleanup.
0002-Remove-js_quotes-union-entry.patch
We usually don't want to put every single node type into the gram.y
%union. This one can be trivially removed.
0003-Move-some-code-from-gram.y-to-parse-analysis.patch
Code like this can be postponed to parse analysis, keeping gram.y
smaller. The error pointer loses a bit of precision, but I think that's
ok. (There is similar code in your 0004 patch, which could be similarly
moved.)
0004-Remove-JsonBehavior-stuff-from-union.patch
Similar to my 0002. This adds a few casts as a result, but that is the
typical style in gram.y.
0005-Get-rid-of-JsonBehaviorClause.patch
I think this two-level wrapping of the behavior clauses is both
confusing and overkill. I was trying to just list the on-empty and
on-error clauses separately in the top-level productions (JSON_VALUE
etc.), but that led to shift/reduce errors. So the existing rule
structure is probably ok. But we don't need a separate node type just
to combine two values and then unpack them again shortly thereafter. So
I just replaced all this with a list.
0006-Get-rid-of-JsonCommon.patch
This is an example where the SQL standard BNF is not sensible to apply
literally. I moved those clauses up directly into their callers, thus
removing one intermediate levels of rules and also nodes. Also, the
path name (AS name) stuff is only for JSON_TABLE, so it's not needed in
this patch. I removed it here, but it would have to be readded in your
0004 patch.
Another thing: In your patch, JSON_EXISTS has a RETURNING clause
(json_returning_clause_opt), but I don't see that in the standard, and
also not in the Oracle or Db2 docs. Where did this come from?
With these changes, I think the grammar complexity in your 0003 patch is
at an acceptable level. Similar simplification opportunities exist in
the 0004 patch, but I haven't worked on that yet. I suggest that you
focus on getting 0001..0003 committed around this commit fest and then
deal with 0004 in the next one. (Also split up the 0005 patch into the
pieces that apply to 0003 and 0004, respectively.)
From 90cd46c91231a29a41118861d5a6122d78f93379 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:19:03 +0100
Subject: [PATCH 1/6] Put keywords in right order
---
src/backend/parser/gram.y | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 1dc3300fde..9a7058b767 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -735,7 +735,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
- KEY KEYS KEEP
+ KEEP KEY KEYS
LABEL LANGUAGE LARGE_P LAST_P LATERAL_P
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
--
2.42.1
From b669a45c9603a23db240cf1566b3f2e726254ac4 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:28:17 +0100
Subject: [PATCH 2/6] Remove js_quotes %union entry
---
src/backend/parser/gram.y | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 9a7058b767..a8cce5b00e 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -280,7 +280,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
struct KeyAction *keyaction;
JsonBehavior *jsbehavior;
JsonBehaviorClause *jsbehaviorclause;
- JsonQuotes js_quotes;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -662,12 +661,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
+ json_quotes_clause_opt
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
%type <jsbehavior> json_behavior
%type <jsbehaviorclause> json_behavior_clause_opt
-%type <js_quotes> json_quotes_clause_opt
/*
--
2.42.1
From 7fb1906bab90e539697e6d66d3f2754eb3031603 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:43:10 +0100
Subject: [PATCH 3/6] Move some code from gram.y to parse analysis
---
src/backend/parser/gram.y | 5 -----
src/backend/parser/parse_expr.c | 6 ++++++
src/test/regress/expected/jsonb_sqljson.out | 12 ++++++------
3 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index a8cce5b00e..a2d482ab70 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -15778,11 +15778,6 @@ func_expr_common_subexpr:
n->common = (JsonCommon *) $3;
n->output = (JsonOutput *) $4;
n->wrapper = $5;
- if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
- ereport(ERROR,
- (errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
- parser_errposition(@6)));
n->quotes = $6;
n->behavior = $7;
n->location = @1;
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64571562c5..1c1f38a808 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4302,6 +4302,12 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
case JSON_QUERY_OP:
func_name = "JSON_QUERY";
+ if (func->wrapper != JSW_NONE && func->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(pstate, func->location)));
+
jsexpr = transformJsonExprCommon(pstate, func, func_name);
jsexpr->wrapper = func->wrapper;
jsexpr->omit_quotes = (func->quotes == JS_QUOTES_OMIT);
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
index b467971ed9..9bf3fb21db 100644
--- a/src/test/regress/expected/jsonb_sqljson.out
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -658,19 +658,19 @@ SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERR
SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES)...
- ^
+ ^
SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES)...
- ^
+ ^
SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
-LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTE...
- ^
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER ...
+ ^
SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
-LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTE...
- ^
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER ...
+ ^
-- Should succeed
SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
json_query
--
2.42.1
From a7e78b5e9c5cfa191a9580a7b98f6ea663edaa36 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:53:55 +0100
Subject: [PATCH 4/6] Remove JsonBehavior stuff from %union
---
src/backend/parser/gram.y | 36 +++++++++++++++++-------------------
1 file changed, 17 insertions(+), 19 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index a2d482ab70..2866d1345c 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -278,8 +278,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
- JsonBehavior *jsbehavior;
- JsonBehaviorClause *jsbehaviorclause;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -654,6 +652,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_aggregate_func
json_api_common_syntax
json_argument
+ json_behavior
+ json_behavior_clause_opt
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
@@ -665,8 +665,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-%type <jsbehavior> json_behavior
-%type <jsbehaviorclause> json_behavior_clause_opt
/*
@@ -15779,7 +15777,7 @@ func_expr_common_subexpr:
n->output = (JsonOutput *) $4;
n->wrapper = $5;
n->quotes = $6;
- n->behavior = $7;
+ n->behavior = (JsonBehaviorClause *) $7;
n->location = @1;
$$ = (Node *) n;
}
@@ -15794,7 +15792,7 @@ func_expr_common_subexpr:
p->op = JSON_EXISTS_OP;
p->common = (JsonCommon *) $3;
p->output = (JsonOutput *) $4;
- p->behavior = $5;
+ p->behavior = (JsonBehaviorClause *) $5;
p->location = @1;
$$ = (Node *) p;
}
@@ -15809,7 +15807,7 @@ func_expr_common_subexpr:
n->op = JSON_VALUE_OP;
n->common = (JsonCommon *) $3;
n->output = (JsonOutput *) $4;
- n->behavior = $5;
+ n->behavior = (JsonBehaviorClause *) $5;
n->location = @1;
$$ = (Node *) n;
}
@@ -16665,25 +16663,25 @@ json_returning_clause_opt:
;
json_behavior:
- DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
- | ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
- | NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
- | TRUE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
- | FALSE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
- | UNKNOWN { $$ = makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
- | EMPTY_P ARRAY { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
- | EMPTY_P OBJECT_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
+ DEFAULT a_expr { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
+ | ERROR_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | NULL_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
+ | TRUE_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
+ | FALSE_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
+ | UNKNOWN { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
+ | EMPTY_P ARRAY { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P OBJECT_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
/* non-standard, for Oracle compatibility only */
- | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
;
json_behavior_clause_opt:
json_behavior ON EMPTY_P
- { $$ = makeJsonBehaviorClause($1, NULL, @1); }
+ { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, NULL, @1); }
| json_behavior ON ERROR_P
- { $$ = makeJsonBehaviorClause(NULL, $1, @1); }
+ { $$ = (Node *) makeJsonBehaviorClause(NULL, (JsonBehavior *) $1, @1); }
| json_behavior ON EMPTY_P json_behavior ON ERROR_P
- { $$ = makeJsonBehaviorClause($1, $4, @1); }
+ { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, (JsonBehavior *) $4, @1); }
| /* EMPTY */
{ $$ = NULL; }
;
--
2.42.1
From 34795db318b16892926dae9ebc848827771136fa Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 07:21:03 +0100
Subject: [PATCH 5/6] Get rid of JsonBehaviorClause
---
src/backend/nodes/makefuncs.c | 17 -----------------
src/backend/nodes/nodeFuncs.c | 10 ----------
src/backend/parser/gram.y | 16 ++++++++--------
src/backend/parser/parse_expr.c | 4 ++--
src/include/nodes/makefuncs.h | 3 ---
src/include/nodes/parsenodes.h | 14 +-------------
src/tools/pgindent/typedefs.list | 1 -
7 files changed, 11 insertions(+), 54 deletions(-)
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index 746ed3d1da..a3757af6cf 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -857,23 +857,6 @@ makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
return jve;
}
-/*
- * makeJsonBehaviorClause -
- * creates a JsonBehaviorClause node
- */
-JsonBehaviorClause *
-makeJsonBehaviorClause(JsonBehavior *on_empty, JsonBehavior *on_error,
- int location)
-{
- JsonBehaviorClause *behavior = makeNode(JsonBehaviorClause);
-
- behavior->on_empty = on_empty;
- behavior->on_error = on_error;
- behavior->location = location;
-
- return behavior;
-}
-
/*
* makeJsonBehavior -
* creates a JsonBehavior node
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 8e49ef6b95..7cad662e28 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -4093,16 +4093,6 @@ raw_expression_tree_walker_impl(Node *node,
return true;
}
break;
- case T_JsonBehaviorClause:
- {
- JsonBehaviorClause *jbc = (JsonBehaviorClause *) node;
-
- if (WALK(jbc->on_empty))
- return true;
- if (WALK(jbc->on_error))
- return true;
- }
- break;
case T_JsonBehavior:
{
JsonBehavior *jb = (JsonBehavior *) node;
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 2866d1345c..99585308c8 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -653,11 +653,11 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_api_common_syntax
json_argument
json_behavior
- json_behavior_clause_opt
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
json_arguments
+ json_behavior_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
@@ -15777,7 +15777,7 @@ func_expr_common_subexpr:
n->output = (JsonOutput *) $4;
n->wrapper = $5;
n->quotes = $6;
- n->behavior = (JsonBehaviorClause *) $7;
+ n->behavior = $7;
n->location = @1;
$$ = (Node *) n;
}
@@ -15792,7 +15792,7 @@ func_expr_common_subexpr:
p->op = JSON_EXISTS_OP;
p->common = (JsonCommon *) $3;
p->output = (JsonOutput *) $4;
- p->behavior = (JsonBehaviorClause *) $5;
+ p->behavior = $5;
p->location = @1;
$$ = (Node *) p;
}
@@ -15807,7 +15807,7 @@ func_expr_common_subexpr:
n->op = JSON_VALUE_OP;
n->common = (JsonCommon *) $3;
n->output = (JsonOutput *) $4;
- n->behavior = (JsonBehaviorClause *) $5;
+ n->behavior = $5;
n->location = @1;
$$ = (Node *) n;
}
@@ -16677,13 +16677,13 @@ json_behavior:
json_behavior_clause_opt:
json_behavior ON EMPTY_P
- { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, NULL, @1); }
+ { $$ = list_make2($1, NULL); }
| json_behavior ON ERROR_P
- { $$ = (Node *) makeJsonBehaviorClause(NULL, (JsonBehavior *) $1, @1); }
+ { $$ = list_make2(NULL, $1); }
| json_behavior ON EMPTY_P json_behavior ON ERROR_P
- { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, (JsonBehavior *) $4, @1); }
+ { $$ = list_make2($1, $4); }
| /* EMPTY */
- { $$ = NULL; }
+ { $$ = list_make2(NULL, NULL); }
;
json_predicate_type_constraint:
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 1c1f38a808..2f4d3cb118 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4240,8 +4240,8 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
if (func->behavior)
{
- on_error = func->behavior->on_error;
- on_empty = func->behavior->on_empty;
+ on_empty = linitial(func->behavior);
+ on_error = lsecond(func->behavior);
}
switch (func->op)
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 783e82ac92..a850a1928b 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -113,9 +113,6 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
-extern JsonBehaviorClause *makeJsonBehaviorClause(JsonBehavior *on_empty,
- JsonBehavior *on_error,
- int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ce6893fb91..0f977f935e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1703,18 +1703,6 @@ typedef enum JsonQuotes
JS_QUOTES_OMIT /* OMIT QUOTES */
} JsonQuotes;
-/*
- * JsonBehaviorClause -
- * representation of JSON ON ERROR / ON EMPTY clause
- */
-typedef struct JsonBehaviorClause
-{
- NodeTag type;
- JsonBehavior *on_empty; /* behavior ON EMPTY */
- JsonBehavior *on_error; /* behavior ON ERROR */
- int location; /* token location, or -1 if unknown */
-} JsonBehaviorClause;
-
/*
* JsonPathSpec -
* representation of JSON path constant
@@ -1767,7 +1755,7 @@ typedef struct JsonFuncExpr
JsonExprOp op; /* expression type */
JsonCommon *common; /* common syntax */
JsonOutput *output; /* output clause, if specified */
- JsonBehaviorClause *behavior; /* ON ERROR / EMPTY behavior, if specified */
+ List *behavior; /* ON EMPTY / ERROR behavior, list of two JsonBehavior nodes */
JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
JsonQuotes quotes; /* omit or keep quotes? (JSON_QUERY only) */
int location; /* token location, or -1 if unknown */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 3cfa9d756e..9258576b4c 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1260,7 +1260,6 @@ JsonArrayConstructor
JsonArrayQueryConstructor
JsonBaseObjectInfo
JsonBehavior
-JsonBehaviorClause
JsonBehaviorType
JsonCoercion
JsonCoercionState
--
2.42.1
From 87f6b4ad2d85a243b106c933c3f8830e5f64451f Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 07:25:41 +0100
Subject: [PATCH 6/6] Get rid of JsonCommon
---
src/backend/nodes/nodeFuncs.c | 18 ++----
src/backend/parser/gram.y | 95 ++++++++++----------------------
src/backend/parser/parse_expr.c | 8 +--
src/include/nodes/parsenodes.h | 18 +-----
src/tools/pgindent/typedefs.list | 1 -
5 files changed, 40 insertions(+), 100 deletions(-)
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 7cad662e28..b6cd8c38d2 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -4081,18 +4081,6 @@ raw_expression_tree_walker_impl(Node *node,
return WALK(((JsonIsPredicate *) node)->expr);
case T_JsonArgument:
return WALK(((JsonArgument *) node)->val);
- case T_JsonCommon:
- {
- JsonCommon *jc = (JsonCommon *) node;
-
- if (WALK(jc->expr))
- return true;
- if (WALK(jc->pathspec))
- return true;
- if (WALK(jc->passing))
- return true;
- }
- break;
case T_JsonBehavior:
{
JsonBehavior *jb = (JsonBehavior *) node;
@@ -4106,7 +4094,11 @@ raw_expression_tree_walker_impl(Node *node,
{
JsonFuncExpr *jfe = (JsonFuncExpr *) node;
- if (WALK(jfe->common))
+ if (WALK(jfe->context_item))
+ return true;
+ if (WALK(jfe->pathspec))
+ return true;
+ if (WALK(jfe->passing))
return true;
if (jfe->output && WALK(jfe->output))
return true;
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 99585308c8..4cdf086a42 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -650,7 +650,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_returning_clause_opt
json_name_and_value
json_aggregate_func
- json_api_common_syntax
json_argument
json_behavior
%type <list> json_name_and_value_list
@@ -658,6 +657,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_array_aggregate_order_by_clause_opt
json_arguments
json_behavior_clause_opt
+ json_passing_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
@@ -15763,7 +15763,7 @@ func_expr_common_subexpr:
$$ = (Node *) n;
}
| JSON_QUERY '('
- json_api_common_syntax
+ json_value_expr ',' a_expr json_passing_clause_opt
json_returning_clause_opt
json_wrapper_behavior
json_quotes_clause_opt
@@ -15773,31 +15773,35 @@ func_expr_common_subexpr:
JsonFuncExpr *n = makeNode(JsonFuncExpr);
n->op = JSON_QUERY_OP;
- n->common = (JsonCommon *) $3;
- n->output = (JsonOutput *) $4;
- n->wrapper = $5;
- n->quotes = $6;
- n->behavior = $7;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->wrapper = $8;
+ n->quotes = $9;
+ n->behavior = $10;
n->location = @1;
$$ = (Node *) n;
}
| JSON_EXISTS '('
- json_api_common_syntax
+ json_value_expr ',' a_expr json_passing_clause_opt
json_returning_clause_opt
json_behavior_clause_opt
')'
{
- JsonFuncExpr *p = makeNode(JsonFuncExpr);
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
- p->op = JSON_EXISTS_OP;
- p->common = (JsonCommon *) $3;
- p->output = (JsonOutput *) $4;
- p->behavior = $5;
- p->location = @1;
- $$ = (Node *) p;
+ n->op = JSON_EXISTS_OP;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->behavior = $8;
+ n->location = @1;
+ $$ = (Node *) n;
}
| JSON_VALUE '('
- json_api_common_syntax
+ json_value_expr ',' a_expr json_passing_clause_opt
json_returning_clause_opt
json_behavior_clause_opt
')'
@@ -15805,9 +15809,11 @@ func_expr_common_subexpr:
JsonFuncExpr *n = makeNode(JsonFuncExpr);
n->op = JSON_VALUE_OP;
- n->common = (JsonCommon *) $3;
- n->output = (JsonOutput *) $4;
- n->behavior = $5;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->behavior = $8;
n->location = @1;
$$ = (Node *) n;
}
@@ -16537,54 +16543,9 @@ opt_asymmetric: ASYMMETRIC
;
/* SQL/JSON support */
-json_api_common_syntax:
- json_value_expr ',' a_expr /* i.e. a json_path */
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = NULL;
- n->passing = NULL;
- n->location = @1;
- $$ = (Node *) n;
- }
- | json_value_expr ',' a_expr AS name
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = $5;
- n->passing = NULL;
- n->location = @1;
- $$ = (Node *) n;
- }
-
- | json_value_expr ',' a_expr /* i.e. a json_path */
- PASSING json_arguments
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = NULL;
- n->passing = $5;
- n->location = @1;
- $$ = (Node *) n;
- }
- | json_value_expr ',' a_expr AS name
- PASSING json_arguments
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = $5;
- n->passing = $7;
- n->location = @1;
- $$ = (Node *) n;
- }
+json_passing_clause_opt:
+ PASSING json_arguments { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
;
json_arguments:
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 2f4d3cb118..af0843aa3c 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4384,17 +4384,17 @@ transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
jsexpr->location = func->location;
jsexpr->op = func->op;
jsexpr->formatted_expr = transformJsonValueExpr(pstate, constructName,
- func->common->expr,
+ func->context_item,
JS_FORMAT_JSON,
InvalidOid, false);
- jsexpr->format = func->common->expr->format;
+ jsexpr->format = func->context_item->format;
/* Both set in the caller. */
jsexpr->result_coercion = NULL;
jsexpr->omit_quotes = false;
- pathspec = transformExprRecurse(pstate, func->common->pathspec);
+ pathspec = transformExprRecurse(pstate, func->pathspec);
jsexpr->path_spec =
coerce_to_target_type(pstate, pathspec, exprType(pathspec),
@@ -4415,7 +4415,7 @@ transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
transformJsonPassingArgs(pstate, constructName,
exprType(jsexpr->formatted_expr) == JSONBOID ?
JS_FORMAT_JSONB : JS_FORMAT_JSON,
- func->common->passing,
+ func->passing,
&jsexpr->passing_values,
&jsexpr->passing_names);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 0f977f935e..83acd17ad4 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1731,20 +1731,6 @@ typedef struct JsonArgument
char *name; /* argument name */
} JsonArgument;
-/*
- * JsonCommon -
- * representation of common syntax of functions using JSON path
- */
-typedef struct JsonCommon
-{
- NodeTag type;
- JsonValueExpr *expr; /* context item expression */
- Node *pathspec; /* JSON path specification expression */
- char *pathname; /* path name, if any */
- List *passing; /* list of PASSING clause arguments, if any */
- int location; /* token location, or -1 if unknown */
-} JsonCommon;
-
/*
* JsonFuncExpr -
* untransformed representation of JSON function expressions
@@ -1753,7 +1739,9 @@ typedef struct JsonFuncExpr
{
NodeTag type;
JsonExprOp op; /* expression type */
- JsonCommon *common; /* common syntax */
+ JsonValueExpr *context_item; /* context item expression */
+ Node *pathspec; /* JSON path specification expression */
+ List *passing; /* list of PASSING clause arguments, if any */
JsonOutput *output; /* output clause, if specified */
List *behavior; /* ON EMPTY / ERROR behavior, list of two JsonBehavior nodes */
JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9258576b4c..8186a42659 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1263,7 +1263,6 @@ JsonBehavior
JsonBehaviorType
JsonCoercion
JsonCoercionState
-JsonCommon
JsonConstructorExpr
JsonConstructorExprState
JsonConstructorType
--
2.42.1
Attachments:
[text/plain] 0001-Put-keywords-in-right-order.patch.nocfbot (842B, ../../[email protected]/2-0001-Put-keywords-in-right-order.patch.nocfbot)
download | inline diff:
From 90cd46c91231a29a41118861d5a6122d78f93379 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:19:03 +0100
Subject: [PATCH 1/6] Put keywords in right order
---
src/backend/parser/gram.y | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 1dc3300fde..9a7058b767 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -735,7 +735,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
JOIN JSON JSON_ARRAY JSON_ARRAYAGG JSON_EXISTS JSON_OBJECT JSON_OBJECTAGG
JSON_QUERY JSON_SCALAR JSON_SERIALIZE JSON_VALUE
- KEY KEYS KEEP
+ KEEP KEY KEYS
LABEL LANGUAGE LARGE_P LAST_P LATERAL_P
LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
--
2.42.1
[text/plain] 0002-Remove-js_quotes-union-entry.patch.nocfbot (1.2K, ../../[email protected]/3-0002-Remove-js_quotes-union-entry.patch.nocfbot)
download | inline diff:
From b669a45c9603a23db240cf1566b3f2e726254ac4 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:28:17 +0100
Subject: [PATCH 2/6] Remove js_quotes %union entry
---
src/backend/parser/gram.y | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 9a7058b767..a8cce5b00e 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -280,7 +280,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
struct KeyAction *keyaction;
JsonBehavior *jsbehavior;
JsonBehaviorClause *jsbehaviorclause;
- JsonQuotes js_quotes;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -662,12 +661,12 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
+ json_quotes_clause_opt
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
%type <jsbehavior> json_behavior
%type <jsbehaviorclause> json_behavior_clause_opt
-%type <js_quotes> json_quotes_clause_opt
/*
--
2.42.1
[text/plain] 0003-Move-some-code-from-gram.y-to-parse-analysis.patch.nocfbot (3.6K, ../../[email protected]/4-0003-Move-some-code-from-gram.y-to-parse-analysis.patch.nocfbot)
download | inline diff:
From 7fb1906bab90e539697e6d66d3f2754eb3031603 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:43:10 +0100
Subject: [PATCH 3/6] Move some code from gram.y to parse analysis
---
src/backend/parser/gram.y | 5 -----
src/backend/parser/parse_expr.c | 6 ++++++
src/test/regress/expected/jsonb_sqljson.out | 12 ++++++------
3 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index a8cce5b00e..a2d482ab70 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -15778,11 +15778,6 @@ func_expr_common_subexpr:
n->common = (JsonCommon *) $3;
n->output = (JsonOutput *) $4;
n->wrapper = $5;
- if (n->wrapper != JSW_NONE && $6 != JS_QUOTES_UNSPEC)
- ereport(ERROR,
- (errcode(ERRCODE_SYNTAX_ERROR),
- errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
- parser_errposition(@6)));
n->quotes = $6;
n->behavior = $7;
n->location = @1;
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 64571562c5..1c1f38a808 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4302,6 +4302,12 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
case JSON_QUERY_OP:
func_name = "JSON_QUERY";
+ if (func->wrapper != JSW_NONE && func->quotes != JS_QUOTES_UNSPEC)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used"),
+ parser_errposition(pstate, func->location)));
+
jsexpr = transformJsonExprCommon(pstate, func, func_name);
jsexpr->wrapper = func->wrapper;
jsexpr->omit_quotes = (func->quotes == JS_QUOTES_OMIT);
diff --git a/src/test/regress/expected/jsonb_sqljson.out b/src/test/regress/expected/jsonb_sqljson.out
index b467971ed9..9bf3fb21db 100644
--- a/src/test/regress/expected/jsonb_sqljson.out
+++ b/src/test/regress/expected/jsonb_sqljson.out
@@ -658,19 +658,19 @@ SELECT JSON_QUERY(jsonb '"aaa"', '$' RETURNING bytea FORMAT JSON OMIT QUOTES ERR
SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER OMIT QUOTES)...
- ^
+ ^
SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH WRAPPER KEEP QUOTES)...
- ^
+ ^
SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
-LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER KEEP QUOTE...
- ^
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER ...
+ ^
SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTES);
ERROR: SQL/JSON QUOTES behavior must not be specified when WITH WRAPPER is used
-LINE 1: ...N_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER OMIT QUOTE...
- ^
+LINE 1: SELECT JSON_QUERY(jsonb '[1]', '$' WITH CONDITIONAL WRAPPER ...
+ ^
-- Should succeed
SELECT JSON_QUERY(jsonb '[1]', '$' WITHOUT WRAPPER OMIT QUOTES);
json_query
--
2.42.1
[text/plain] 0004-Remove-JsonBehavior-stuff-from-union.patch.nocfbot (4.4K, ../../[email protected]/5-0004-Remove-JsonBehavior-stuff-from-union.patch.nocfbot)
download | inline diff:
From a7e78b5e9c5cfa191a9580a7b98f6ea663edaa36 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 05:53:55 +0100
Subject: [PATCH 4/6] Remove JsonBehavior stuff from %union
---
src/backend/parser/gram.y | 36 +++++++++++++++++-------------------
1 file changed, 17 insertions(+), 19 deletions(-)
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index a2d482ab70..2866d1345c 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -278,8 +278,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
MergeWhenClause *mergewhen;
struct KeyActions *keyactions;
struct KeyAction *keyaction;
- JsonBehavior *jsbehavior;
- JsonBehaviorClause *jsbehaviorclause;
}
%type <node> stmt toplevel_stmt schema_stmt routine_body_stmt
@@ -654,6 +652,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_aggregate_func
json_api_common_syntax
json_argument
+ json_behavior
+ json_behavior_clause_opt
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
@@ -665,8 +665,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
%type <boolean> json_key_uniqueness_constraint_opt
json_object_constructor_null_clause_opt
json_array_constructor_null_clause_opt
-%type <jsbehavior> json_behavior
-%type <jsbehaviorclause> json_behavior_clause_opt
/*
@@ -15779,7 +15777,7 @@ func_expr_common_subexpr:
n->output = (JsonOutput *) $4;
n->wrapper = $5;
n->quotes = $6;
- n->behavior = $7;
+ n->behavior = (JsonBehaviorClause *) $7;
n->location = @1;
$$ = (Node *) n;
}
@@ -15794,7 +15792,7 @@ func_expr_common_subexpr:
p->op = JSON_EXISTS_OP;
p->common = (JsonCommon *) $3;
p->output = (JsonOutput *) $4;
- p->behavior = $5;
+ p->behavior = (JsonBehaviorClause *) $5;
p->location = @1;
$$ = (Node *) p;
}
@@ -15809,7 +15807,7 @@ func_expr_common_subexpr:
n->op = JSON_VALUE_OP;
n->common = (JsonCommon *) $3;
n->output = (JsonOutput *) $4;
- n->behavior = $5;
+ n->behavior = (JsonBehaviorClause *) $5;
n->location = @1;
$$ = (Node *) n;
}
@@ -16665,25 +16663,25 @@ json_returning_clause_opt:
;
json_behavior:
- DEFAULT a_expr { $$ = makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
- | ERROR_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
- | NULL_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
- | TRUE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
- | FALSE_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
- | UNKNOWN { $$ = makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
- | EMPTY_P ARRAY { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
- | EMPTY_P OBJECT_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
+ DEFAULT a_expr { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_DEFAULT, $2, @1); }
+ | ERROR_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_ERROR, NULL, @1); }
+ | NULL_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_NULL, NULL, @1); }
+ | TRUE_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_TRUE, NULL, @1); }
+ | FALSE_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_FALSE, NULL, @1); }
+ | UNKNOWN { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_UNKNOWN, NULL, @1); }
+ | EMPTY_P ARRAY { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P OBJECT_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_OBJECT, NULL, @1); }
/* non-standard, for Oracle compatibility only */
- | EMPTY_P { $$ = makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
+ | EMPTY_P { $$ = (Node *) makeJsonBehavior(JSON_BEHAVIOR_EMPTY_ARRAY, NULL, @1); }
;
json_behavior_clause_opt:
json_behavior ON EMPTY_P
- { $$ = makeJsonBehaviorClause($1, NULL, @1); }
+ { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, NULL, @1); }
| json_behavior ON ERROR_P
- { $$ = makeJsonBehaviorClause(NULL, $1, @1); }
+ { $$ = (Node *) makeJsonBehaviorClause(NULL, (JsonBehavior *) $1, @1); }
| json_behavior ON EMPTY_P json_behavior ON ERROR_P
- { $$ = makeJsonBehaviorClause($1, $4, @1); }
+ { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, (JsonBehavior *) $4, @1); }
| /* EMPTY */
{ $$ = NULL; }
;
--
2.42.1
[text/plain] 0005-Get-rid-of-JsonBehaviorClause.patch.nocfbot (6.8K, ../../[email protected]/6-0005-Get-rid-of-JsonBehaviorClause.patch.nocfbot)
download | inline diff:
From 34795db318b16892926dae9ebc848827771136fa Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 07:21:03 +0100
Subject: [PATCH 5/6] Get rid of JsonBehaviorClause
---
src/backend/nodes/makefuncs.c | 17 -----------------
src/backend/nodes/nodeFuncs.c | 10 ----------
src/backend/parser/gram.y | 16 ++++++++--------
src/backend/parser/parse_expr.c | 4 ++--
src/include/nodes/makefuncs.h | 3 ---
src/include/nodes/parsenodes.h | 14 +-------------
src/tools/pgindent/typedefs.list | 1 -
7 files changed, 11 insertions(+), 54 deletions(-)
diff --git a/src/backend/nodes/makefuncs.c b/src/backend/nodes/makefuncs.c
index 746ed3d1da..a3757af6cf 100644
--- a/src/backend/nodes/makefuncs.c
+++ b/src/backend/nodes/makefuncs.c
@@ -857,23 +857,6 @@ makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
return jve;
}
-/*
- * makeJsonBehaviorClause -
- * creates a JsonBehaviorClause node
- */
-JsonBehaviorClause *
-makeJsonBehaviorClause(JsonBehavior *on_empty, JsonBehavior *on_error,
- int location)
-{
- JsonBehaviorClause *behavior = makeNode(JsonBehaviorClause);
-
- behavior->on_empty = on_empty;
- behavior->on_error = on_error;
- behavior->location = location;
-
- return behavior;
-}
-
/*
* makeJsonBehavior -
* creates a JsonBehavior node
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 8e49ef6b95..7cad662e28 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -4093,16 +4093,6 @@ raw_expression_tree_walker_impl(Node *node,
return true;
}
break;
- case T_JsonBehaviorClause:
- {
- JsonBehaviorClause *jbc = (JsonBehaviorClause *) node;
-
- if (WALK(jbc->on_empty))
- return true;
- if (WALK(jbc->on_error))
- return true;
- }
- break;
case T_JsonBehavior:
{
JsonBehavior *jb = (JsonBehavior *) node;
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 2866d1345c..99585308c8 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -653,11 +653,11 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_api_common_syntax
json_argument
json_behavior
- json_behavior_clause_opt
%type <list> json_name_and_value_list
json_value_expr_list
json_array_aggregate_order_by_clause_opt
json_arguments
+ json_behavior_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
@@ -15777,7 +15777,7 @@ func_expr_common_subexpr:
n->output = (JsonOutput *) $4;
n->wrapper = $5;
n->quotes = $6;
- n->behavior = (JsonBehaviorClause *) $7;
+ n->behavior = $7;
n->location = @1;
$$ = (Node *) n;
}
@@ -15792,7 +15792,7 @@ func_expr_common_subexpr:
p->op = JSON_EXISTS_OP;
p->common = (JsonCommon *) $3;
p->output = (JsonOutput *) $4;
- p->behavior = (JsonBehaviorClause *) $5;
+ p->behavior = $5;
p->location = @1;
$$ = (Node *) p;
}
@@ -15807,7 +15807,7 @@ func_expr_common_subexpr:
n->op = JSON_VALUE_OP;
n->common = (JsonCommon *) $3;
n->output = (JsonOutput *) $4;
- n->behavior = (JsonBehaviorClause *) $5;
+ n->behavior = $5;
n->location = @1;
$$ = (Node *) n;
}
@@ -16677,13 +16677,13 @@ json_behavior:
json_behavior_clause_opt:
json_behavior ON EMPTY_P
- { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, NULL, @1); }
+ { $$ = list_make2($1, NULL); }
| json_behavior ON ERROR_P
- { $$ = (Node *) makeJsonBehaviorClause(NULL, (JsonBehavior *) $1, @1); }
+ { $$ = list_make2(NULL, $1); }
| json_behavior ON EMPTY_P json_behavior ON ERROR_P
- { $$ = (Node *) makeJsonBehaviorClause((JsonBehavior *) $1, (JsonBehavior *) $4, @1); }
+ { $$ = list_make2($1, $4); }
| /* EMPTY */
- { $$ = NULL; }
+ { $$ = list_make2(NULL, NULL); }
;
json_predicate_type_constraint:
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 1c1f38a808..2f4d3cb118 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4240,8 +4240,8 @@ transformJsonFuncExpr(ParseState *pstate, JsonFuncExpr *func)
if (func->behavior)
{
- on_error = func->behavior->on_error;
- on_empty = func->behavior->on_empty;
+ on_empty = linitial(func->behavior);
+ on_error = lsecond(func->behavior);
}
switch (func->op)
diff --git a/src/include/nodes/makefuncs.h b/src/include/nodes/makefuncs.h
index 783e82ac92..a850a1928b 100644
--- a/src/include/nodes/makefuncs.h
+++ b/src/include/nodes/makefuncs.h
@@ -113,9 +113,6 @@ extern JsonFormat *makeJsonFormat(JsonFormatType type, JsonEncoding encoding,
extern JsonValueExpr *makeJsonValueExpr(Expr *raw_expr, Expr *formatted_expr,
JsonFormat *format);
extern JsonBehavior *makeJsonBehavior(JsonBehaviorType type, Node *expr, int location);
-extern JsonBehaviorClause *makeJsonBehaviorClause(JsonBehavior *on_empty,
- JsonBehavior *on_error,
- int location);
extern Node *makeJsonKeyValue(Node *key, Node *value);
extern Node *makeJsonIsPredicate(Node *expr, JsonFormat *format,
JsonValueType item_type, bool unique_keys,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index ce6893fb91..0f977f935e 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1703,18 +1703,6 @@ typedef enum JsonQuotes
JS_QUOTES_OMIT /* OMIT QUOTES */
} JsonQuotes;
-/*
- * JsonBehaviorClause -
- * representation of JSON ON ERROR / ON EMPTY clause
- */
-typedef struct JsonBehaviorClause
-{
- NodeTag type;
- JsonBehavior *on_empty; /* behavior ON EMPTY */
- JsonBehavior *on_error; /* behavior ON ERROR */
- int location; /* token location, or -1 if unknown */
-} JsonBehaviorClause;
-
/*
* JsonPathSpec -
* representation of JSON path constant
@@ -1767,7 +1755,7 @@ typedef struct JsonFuncExpr
JsonExprOp op; /* expression type */
JsonCommon *common; /* common syntax */
JsonOutput *output; /* output clause, if specified */
- JsonBehaviorClause *behavior; /* ON ERROR / EMPTY behavior, if specified */
+ List *behavior; /* ON EMPTY / ERROR behavior, list of two JsonBehavior nodes */
JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
JsonQuotes quotes; /* omit or keep quotes? (JSON_QUERY only) */
int location; /* token location, or -1 if unknown */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 3cfa9d756e..9258576b4c 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1260,7 +1260,6 @@ JsonArrayConstructor
JsonArrayQueryConstructor
JsonBaseObjectInfo
JsonBehavior
-JsonBehaviorClause
JsonBehaviorType
JsonCoercion
JsonCoercionState
--
2.42.1
[text/plain] 0006-Get-rid-of-JsonCommon.patch.nocfbot (8.7K, ../../[email protected]/7-0006-Get-rid-of-JsonCommon.patch.nocfbot)
download | inline diff:
From 87f6b4ad2d85a243b106c933c3f8830e5f64451f Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <[email protected]>
Date: Tue, 21 Nov 2023 07:25:41 +0100
Subject: [PATCH 6/6] Get rid of JsonCommon
---
src/backend/nodes/nodeFuncs.c | 18 ++----
src/backend/parser/gram.y | 95 ++++++++++----------------------
src/backend/parser/parse_expr.c | 8 +--
src/include/nodes/parsenodes.h | 18 +-----
src/tools/pgindent/typedefs.list | 1 -
5 files changed, 40 insertions(+), 100 deletions(-)
diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c
index 7cad662e28..b6cd8c38d2 100644
--- a/src/backend/nodes/nodeFuncs.c
+++ b/src/backend/nodes/nodeFuncs.c
@@ -4081,18 +4081,6 @@ raw_expression_tree_walker_impl(Node *node,
return WALK(((JsonIsPredicate *) node)->expr);
case T_JsonArgument:
return WALK(((JsonArgument *) node)->val);
- case T_JsonCommon:
- {
- JsonCommon *jc = (JsonCommon *) node;
-
- if (WALK(jc->expr))
- return true;
- if (WALK(jc->pathspec))
- return true;
- if (WALK(jc->passing))
- return true;
- }
- break;
case T_JsonBehavior:
{
JsonBehavior *jb = (JsonBehavior *) node;
@@ -4106,7 +4094,11 @@ raw_expression_tree_walker_impl(Node *node,
{
JsonFuncExpr *jfe = (JsonFuncExpr *) node;
- if (WALK(jfe->common))
+ if (WALK(jfe->context_item))
+ return true;
+ if (WALK(jfe->pathspec))
+ return true;
+ if (WALK(jfe->passing))
return true;
if (jfe->output && WALK(jfe->output))
return true;
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 99585308c8..4cdf086a42 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -650,7 +650,6 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_returning_clause_opt
json_name_and_value
json_aggregate_func
- json_api_common_syntax
json_argument
json_behavior
%type <list> json_name_and_value_list
@@ -658,6 +657,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
json_array_aggregate_order_by_clause_opt
json_arguments
json_behavior_clause_opt
+ json_passing_clause_opt
%type <ival> json_encoding_clause_opt
json_predicate_type_constraint
json_wrapper_behavior
@@ -15763,7 +15763,7 @@ func_expr_common_subexpr:
$$ = (Node *) n;
}
| JSON_QUERY '('
- json_api_common_syntax
+ json_value_expr ',' a_expr json_passing_clause_opt
json_returning_clause_opt
json_wrapper_behavior
json_quotes_clause_opt
@@ -15773,31 +15773,35 @@ func_expr_common_subexpr:
JsonFuncExpr *n = makeNode(JsonFuncExpr);
n->op = JSON_QUERY_OP;
- n->common = (JsonCommon *) $3;
- n->output = (JsonOutput *) $4;
- n->wrapper = $5;
- n->quotes = $6;
- n->behavior = $7;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->wrapper = $8;
+ n->quotes = $9;
+ n->behavior = $10;
n->location = @1;
$$ = (Node *) n;
}
| JSON_EXISTS '('
- json_api_common_syntax
+ json_value_expr ',' a_expr json_passing_clause_opt
json_returning_clause_opt
json_behavior_clause_opt
')'
{
- JsonFuncExpr *p = makeNode(JsonFuncExpr);
+ JsonFuncExpr *n = makeNode(JsonFuncExpr);
- p->op = JSON_EXISTS_OP;
- p->common = (JsonCommon *) $3;
- p->output = (JsonOutput *) $4;
- p->behavior = $5;
- p->location = @1;
- $$ = (Node *) p;
+ n->op = JSON_EXISTS_OP;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->behavior = $8;
+ n->location = @1;
+ $$ = (Node *) n;
}
| JSON_VALUE '('
- json_api_common_syntax
+ json_value_expr ',' a_expr json_passing_clause_opt
json_returning_clause_opt
json_behavior_clause_opt
')'
@@ -15805,9 +15809,11 @@ func_expr_common_subexpr:
JsonFuncExpr *n = makeNode(JsonFuncExpr);
n->op = JSON_VALUE_OP;
- n->common = (JsonCommon *) $3;
- n->output = (JsonOutput *) $4;
- n->behavior = $5;
+ n->context_item = (JsonValueExpr *) $3;
+ n->pathspec = $5;
+ n->passing = $6;
+ n->output = (JsonOutput *) $7;
+ n->behavior = $8;
n->location = @1;
$$ = (Node *) n;
}
@@ -16537,54 +16543,9 @@ opt_asymmetric: ASYMMETRIC
;
/* SQL/JSON support */
-json_api_common_syntax:
- json_value_expr ',' a_expr /* i.e. a json_path */
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = NULL;
- n->passing = NULL;
- n->location = @1;
- $$ = (Node *) n;
- }
- | json_value_expr ',' a_expr AS name
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = $5;
- n->passing = NULL;
- n->location = @1;
- $$ = (Node *) n;
- }
-
- | json_value_expr ',' a_expr /* i.e. a json_path */
- PASSING json_arguments
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = NULL;
- n->passing = $5;
- n->location = @1;
- $$ = (Node *) n;
- }
- | json_value_expr ',' a_expr AS name
- PASSING json_arguments
- {
- JsonCommon *n = makeNode(JsonCommon);
-
- n->expr = (JsonValueExpr *) $1;
- n->pathspec = $3;
- n->pathname = $5;
- n->passing = $7;
- n->location = @1;
- $$ = (Node *) n;
- }
+json_passing_clause_opt:
+ PASSING json_arguments { $$ = $2; }
+ | /*EMPTY*/ { $$ = NIL; }
;
json_arguments:
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index 2f4d3cb118..af0843aa3c 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -4384,17 +4384,17 @@ transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
jsexpr->location = func->location;
jsexpr->op = func->op;
jsexpr->formatted_expr = transformJsonValueExpr(pstate, constructName,
- func->common->expr,
+ func->context_item,
JS_FORMAT_JSON,
InvalidOid, false);
- jsexpr->format = func->common->expr->format;
+ jsexpr->format = func->context_item->format;
/* Both set in the caller. */
jsexpr->result_coercion = NULL;
jsexpr->omit_quotes = false;
- pathspec = transformExprRecurse(pstate, func->common->pathspec);
+ pathspec = transformExprRecurse(pstate, func->pathspec);
jsexpr->path_spec =
coerce_to_target_type(pstate, pathspec, exprType(pathspec),
@@ -4415,7 +4415,7 @@ transformJsonExprCommon(ParseState *pstate, JsonFuncExpr *func,
transformJsonPassingArgs(pstate, constructName,
exprType(jsexpr->formatted_expr) == JSONBOID ?
JS_FORMAT_JSONB : JS_FORMAT_JSON,
- func->common->passing,
+ func->passing,
&jsexpr->passing_values,
&jsexpr->passing_names);
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index 0f977f935e..83acd17ad4 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -1731,20 +1731,6 @@ typedef struct JsonArgument
char *name; /* argument name */
} JsonArgument;
-/*
- * JsonCommon -
- * representation of common syntax of functions using JSON path
- */
-typedef struct JsonCommon
-{
- NodeTag type;
- JsonValueExpr *expr; /* context item expression */
- Node *pathspec; /* JSON path specification expression */
- char *pathname; /* path name, if any */
- List *passing; /* list of PASSING clause arguments, if any */
- int location; /* token location, or -1 if unknown */
-} JsonCommon;
-
/*
* JsonFuncExpr -
* untransformed representation of JSON function expressions
@@ -1753,7 +1739,9 @@ typedef struct JsonFuncExpr
{
NodeTag type;
JsonExprOp op; /* expression type */
- JsonCommon *common; /* common syntax */
+ JsonValueExpr *context_item; /* context item expression */
+ Node *pathspec; /* JSON path specification expression */
+ List *passing; /* list of PASSING clause arguments, if any */
JsonOutput *output; /* output clause, if specified */
List *behavior; /* ON EMPTY / ERROR behavior, list of two JsonBehavior nodes */
JsonWrapper wrapper; /* array wrapper behavior (JSON_QUERY only) */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 9258576b4c..8186a42659 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1263,7 +1263,6 @@ JsonBehavior
JsonBehaviorType
JsonCoercion
JsonCoercionState
-JsonCommon
JsonConstructorExpr
JsonConstructorExprState
JsonConstructorType
--
2.42.1
^ permalink raw reply [nested|flat] 94+ messages in thread
end of thread, other threads:[~2023-12-05 12:25 UTC | newest]
Thread overview: 94+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v7 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v5 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2021-02-24 17:33 [PATCH v6 2/2] Add a --outdated option to reindexdb Julien Rouhaud <[email protected]>
2023-09-29 04:57 Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-02 04:24 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-02 05:26 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-03 13:11 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-04 13:26 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-10-06 21:48 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-15 13:00 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-15 17:11 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-16 08:48 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-17 07:27 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-21 03:52 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-22 19:38 ` Re: remaining sql/json patches Andres Freund <[email protected]>
2023-11-23 06:55 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-23 10:46 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-24 08:41 ` Re: remaining sql/json patches jian he <[email protected]>
2023-11-28 02:00 ` Re: remaining sql/json patches jian he <[email protected]>
2023-12-05 12:25 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-24 09:32 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-16 04:53 ` Re: remaining sql/json patches Erik Rijkers <[email protected]>
2023-11-16 04:57 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-16 06:53 ` Re: remaining sql/json patches Amit Langote <[email protected]>
2023-11-21 07:09 ` Re: remaining sql/json patches Peter Eisentraut <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox