public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v30 05/11] Add Incremental View Maintenance support to psql 25+ messages / 5 participants [nested] [flat]
* [PATCH v30 05/11] Add Incremental View Maintenance support to psql @ 2019-12-20 01:21 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Yugo Nagata @ 2019-12-20 01:21 UTC (permalink / raw) Add tab completion and meta-command output for IVM. --- src/bin/psql/describe.c | 32 +++++++++++++++++++++++++++++++- src/bin/psql/tab-complete.c | 14 +++++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index b6a4eb1d56..3664371aa6 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1570,6 +1570,7 @@ describeOneTableDetails(const char *schemaname, char relpersistence; char relreplident; char *relam; + bool isivm; } tableinfo; bool show_column_details = false; @@ -1582,7 +1583,26 @@ describeOneTableDetails(const char *schemaname, initPQExpBuffer(&tmpbuf); /* Get general table info */ - if (pset.sversion >= 120000) + if (pset.sversion >= 170000) + { + printfPQExpBuffer(&buf, + "SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, " + "c.relhastriggers, c.relrowsecurity, c.relforcerowsecurity, " + "false AS relhasoids, c.relispartition, %s, c.reltablespace, " + "CASE WHEN c.reloftype = 0 THEN '' ELSE c.reloftype::pg_catalog.regtype::pg_catalog.text END, " + "c.relpersistence, c.relreplident, am.amname, " + "c.relisivm\n" + "FROM pg_catalog.pg_class c\n " + "LEFT JOIN pg_catalog.pg_class tc ON (c.reltoastrelid = tc.oid)\n" + "LEFT JOIN pg_catalog.pg_am am ON (c.relam = am.oid)\n" + "WHERE c.oid = '%s';", + (verbose ? + "pg_catalog.array_to_string(c.reloptions || " + "array(select 'toast.' || x from pg_catalog.unnest(tc.reloptions) x), ', ')\n" + : "''"), + oid); + } + else if (pset.sversion >= 120000) { printfPQExpBuffer(&buf, "SELECT c.relchecks, c.relkind, c.relhasindex, c.relhasrules, " @@ -1702,6 +1722,10 @@ describeOneTableDetails(const char *schemaname, (char *) NULL : pg_strdup(PQgetvalue(res, 0, 14)); else tableinfo.relam = NULL; + if (pset.sversion >= 170000) + tableinfo.isivm = strcmp(PQgetvalue(res, 0, 15), "t") == 0; + else + tableinfo.isivm = false; PQclear(res); res = NULL; @@ -3555,6 +3579,12 @@ describeOneTableDetails(const char *schemaname, printfPQExpBuffer(&buf, _("Access method: %s"), tableinfo.relam); printTableAddFooter(&cont, buf.data); } + + /* Incremental view maintance info */ + if (verbose && tableinfo.relkind == RELKIND_MATVIEW && tableinfo.isivm) + { + printTableAddFooter(&cont, _("Incremental view maintenance: yes")); + } } /* reloptions, if verbose */ diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index aa1acf8523..9977df2e28 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1245,6 +1245,7 @@ static const pgsql_thing_t words_after_create[] = { {"FOREIGN TABLE", NULL, NULL, NULL}, {"FUNCTION", NULL, NULL, Query_for_list_of_functions}, {"GROUP", Query_for_list_of_roles}, + {"INCREMENTAL MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews, NULL, THING_NO_DROP | THING_NO_ALTER}, {"INDEX", NULL, NULL, &Query_for_list_of_indexes}, {"LANGUAGE", Query_for_list_of_languages}, {"LARGE OBJECT", NULL, NULL, NULL, NULL, THING_NO_CREATE | THING_NO_DROP}, @@ -3252,7 +3253,7 @@ psql_completion(const char *text, int start, int end) if (HeadMatches("CREATE", "SCHEMA")) COMPLETE_WITH("TABLE", "SEQUENCE"); else - COMPLETE_WITH("TABLE", "SEQUENCE", "MATERIALIZED VIEW"); + COMPLETE_WITH("TABLE", "SEQUENCE", "MATERIALIZED VIEW", "INCREMENTAL MATERIALIZED VIEW"); } /* Complete PARTITION BY with RANGE ( or LIST ( or ... */ else if (TailMatches("PARTITION", "BY")) @@ -3597,13 +3598,16 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH("SELECT"); /* CREATE MATERIALIZED VIEW */ - else if (Matches("CREATE", "MATERIALIZED")) + else if (Matches("CREATE", "MATERIALIZED") || + Matches("CREATE", "INCREMENTAL", "MATERIALIZED")) COMPLETE_WITH("VIEW"); - /* Complete CREATE MATERIALIZED VIEW <name> with AS */ - else if (Matches("CREATE", "MATERIALIZED", "VIEW", MatchAny)) + /* Complete CREATE MATERIALIZED VIEW <name> with AS */ + else if (Matches("CREATE", "MATERIALIZED", "VIEW", MatchAny) || + Matches("CREATE", "INCREMENTAL", "MATERIALIZED", "VIEW", MatchAny)) COMPLETE_WITH("AS"); /* Complete "CREATE MATERIALIZED VIEW <sth> AS with "SELECT" */ - else if (Matches("CREATE", "MATERIALIZED", "VIEW", MatchAny, "AS")) + else if (Matches("CREATE", "MATERIALIZED", "VIEW", MatchAny, "AS") || + Matches("CREATE", "INCREMENTAL", "MATERIALIZED", "VIEW", MatchAny, "AS")) COMPLETE_WITH("SELECT"); /* CREATE EVENT TRIGGER */ -- 2.25.1 --Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt Content-Type: text/x-diff; name="v30-0006-Add-Incremental-View-Maintenance-support.patch" Content-Disposition: attachment; filename="v30-0006-Add-Incremental-View-Maintenance-support.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-07-06 15:58 Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2023-07-06 15:58 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> > On 4 Jul 2023, at 21:08, Nathan Bossart <[email protected]> wrote: > > I put together a rebased version of the patch for cfbot. Thanks for doing that, much appreciated! I was busy looking at other peoples patches and hadn't gotten to my own yet =) > On 13 Mar 2023, at 19:21, Nathan Bossart <[email protected]> wrote: > I noticed that git-am complained when I applied the patch: > > Applying: pg_upgrade: run all data type checks per connection > .git/rebase-apply/patch:1023: new blank line at EOF. > + > warning: 1 line adds whitespace errors. Fixed. > + for (int rowno = 0; rowno < ntups; rowno++) > + { > + found = true; > > It looks like "found" is set unconditionally a few lines above, so I think > this is redundant. Correct, this must've been a leftover from a previous coding that changed. Removed. > Also, I think it would be worth breaking check_for_data_types_usage() into > a few separate functions (or doing some other similar refactoring) to > improve readability. At this point, the function is quite lengthy, and I > count 6 levels of indentation at some lines. It it is pretty big for sure, but it's also IMHO not terribly complicated as it's not really performing any hard to follow logic. I have no issues refactoring it, but trying my hand at I was only making (what I consider) less readable code by having to jump around so I consider it a failure. If you have any suggestions, I would be more than happy to review and incorporate those though. Attached is a v5 with the above fixes and a pgindenting to fix up a few runaway comments and indentations. -- Daniel Gustafsson Attachments: [application/octet-stream] v5-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch (37.1K, ../../[email protected]/2-v5-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch) download | inline diff: From 084f64939eff769d6c9810e2c55dbcd942f79154 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Thu, 6 Jul 2023 17:55:45 +0200 Subject: [PATCH v5] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 589 +++++++++++++++++++++----------- src/bin/pg_upgrade/pg_upgrade.h | 29 +- src/bin/pg_upgrade/version.c | 288 +++------------- 3 files changed, 446 insertions(+), 460 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 64024e3b9e..388555822f 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,14 +24,389 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(ClusterInfo *new_cluster); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static int n_data_types_usage_checks = 7; +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = "Checking for system-defined composite types in user tables", + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + "Your installation contains system-defined composite type(s) in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = "Checking for incompatible \"line\" data type", + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + "your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. you can\n" + "drop the problem columns and restart the upgrade.\n" + "a list of the problem columns is in the file:", + .version_hook = old_9_3_check_for_line_data_type_usage + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = "Checking for reg* data types in user tables", + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + "Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = "Checking for incompatible aclitem data type in user tables", + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:", + .version_hook = check_for_aclitem_data_type_usage + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = "Checking for invalid \"unknown\" user columns", + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = old_9_6_check_for_unknown_data_type_usage + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = "Checking for invalid \"sql_identifier\" user columns", + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = old_11_check_for_sql_identifier_data_type_usage + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = "Checking for incompatible \"jsonb\" data type", + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = check_for_jsonb_9_4_usage + }, +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks * checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + + prep_status("Checking for data type usage"); + + /* Prepare an array to store the results of checks in */ + results = pg_malloc(sizeof(bool) * n_data_types_usage_checks); + memset(results, true, sizeof(*results)); + + prep_status_progress("checking all databases"); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + { + cur_check++; + continue; + } + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", cur_check->status); + appendPQExpBuffer(&report, "\n%s\n %s\n", + cur_check->report_text, output_path); + } + results[checknum] = false; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + cur_check++; + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -100,16 +476,9 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -141,21 +510,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -164,14 +524,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1084,185 +1436,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) check_ok(); } - -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite type(s) in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"aclitem\" data type in user tables"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 3eea0139c7..aeb2c60a89 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -328,6 +328,21 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + /* * Global variables */ @@ -450,19 +465,15 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); +bool check_for_jsonb_9_4_usage(ClusterInfo *cluster); +bool old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); +bool check_for_aclitem_data_type_usage(ClusterInfo *cluster); /* parallel.c */ void parallel_exec_prog(const char *log_file, const char *opt_log_file, diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 403a6d7cfa..b1c5628bd7 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,41 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - -/* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. - */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } - - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } /* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type + * check_for_jsonb_9_4_usage() * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. + * JSONB changed its storage format during 9.4 beta, so check for it. */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +check_for_jsonb_9_4_usage(ClusterInfo *cluster) { - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - free(base_query); - - return found; -} - - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void +bool old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } /* @@ -353,41 +158,20 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void +bool old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) + return true; + + return false; } - /* * report_extension_updates() * Report extensions that should be updated. @@ -459,3 +243,21 @@ report_extension_updates(ClusterInfo *cluster) else check_ok(); } + +/* + * check_for_aclitem_data_type_usage + * + * aclitem changed its storage format in 16, so check for it. + */ +bool +check_for_aclitem_data_type_usage(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; +} -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-07-08 21:43 Nathan Bossart <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Nathan Bossart @ 2023-07-08 21:43 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> Thanks for the new patch. On Thu, Jul 06, 2023 at 05:58:33PM +0200, Daniel Gustafsson wrote: >> On 4 Jul 2023, at 21:08, Nathan Bossart <[email protected]> wrote: >> Also, I think it would be worth breaking check_for_data_types_usage() into >> a few separate functions (or doing some other similar refactoring) to >> improve readability. At this point, the function is quite lengthy, and I >> count 6 levels of indentation at some lines. > > > It it is pretty big for sure, but it's also IMHO not terribly complicated as > it's not really performing any hard to follow logic. > > I have no issues refactoring it, but trying my hand at I was only making (what > I consider) less readable code by having to jump around so I consider it a > failure. If you have any suggestions, I would be more than happy to review and > incorporate those though. I don't have a strong opinion about this. + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } Since "script" will be NULL and "db_used" will be false in the first iteration of the loop, couldn't we move this stuff to before the loop? + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); nitpick: І think the current code has two spaces at the beginning of this format string. Did you mean to remove one of them? + if (script) + { + fclose(script); + script = NULL; + } Won't "script" always be initialized here? If I'm following this code correctly, I think everything except the fclose() can be removed. + cur_check++; I think this is unnecessary since we assign "cur_check" at the beginning of every loop iteration. I see two of these. +static int n_data_types_usage_checks = 7; Can we determine this programmatically so that folks don't need to remember to update it? + /* Prepare an array to store the results of checks in */ + results = pg_malloc(sizeof(bool) * n_data_types_usage_checks); + memset(results, true, sizeof(*results)); IMHO it's a little strange that this is initialized to all "true", only because I think most other Postgres code does the opposite. +bool +check_for_aclitem_data_type_usage(ClusterInfo *cluster) Do you think we should rename these functions to something like "should_check_for_*"? They don't actually do the check, they just tell you whether you should based on the version. In fact, I wonder if we could just add the versions directly to data_types_usage_checks so that we don't need the separate hook functions. -- Nathan Bossart Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-07-10 14:43 Daniel Gustafsson <[email protected]> parent: Nathan Bossart <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2023-07-10 14:43 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> > On 8 Jul 2023, at 23:43, Nathan Bossart <[email protected]> wrote: Thanks for reviewing! > Since "script" will be NULL and "db_used" will be false in the first > iteration of the loop, couldn't we move this stuff to before the loop? We could. It's done this way to match how all the other checks are performing the inner loop for consistency. I think being consistent is better than micro optimizing in non-hot codepaths even though it adds some redundancy. > nitpick: І think the current code has two spaces at the beginning of this > format string. Did you mean to remove one of them? Nice catch, I did not. Fixed. > Won't "script" always be initialized here? If I'm following this code > correctly, I think everything except the fclose() can be removed. You are right that this check is superfluous. This is again an artifact of modelling the code around how the other checks work for consistency. At least I think that's a good characteristic of the code. > I think this is unnecessary since we assign "cur_check" at the beginning of > every loop iteration. I see two of these. Right, this is a pointless leftover from a previous version which used a while() loop, and I had missed removing them. Fixed. > +static int n_data_types_usage_checks = 7; > > Can we determine this programmatically so that folks don't need to remember > to update it? Fair point, I've added a counter loop to the beginning of the check function to calculate it. > IMHO it's a little strange that this is initialized to all "true", only > because I think most other Postgres code does the opposite. Agreed, but it made for a less contrived codepath in knowing when an error has been seen already, to avoid duplicate error output, so I think it's worth it. > Do you think we should rename these functions to something like > "should_check_for_*"? They don't actually do the check, they just tell you > whether you should based on the version. I've been pondering that too, and did a rename now along with moving them all to a single place as well as changing the comments to make it clearer. > In fact, I wonder if we could just add the versions directly to > data_types_usage_checks so that we don't need the separate hook functions. We could, but it would be sort of contrived I think since some check <= and some == while some check the catversion as well (and new ones may have other variants. I think this is the least paint-ourselves-in-a-corner version, if we feel it's needlessly complicated and no other variants are added we can revisit this. -- Daniel Gustafsson Attachments: [application/octet-stream] v6-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch (37.1K, ../../[email protected]/2-v6-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch) download | inline diff: From 723fd82ce174426619dadcb9f2d8b81b3ff532ea Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Thu, 6 Jul 2023 17:55:45 +0200 Subject: [PATCH v6] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 595 +++++++++++++++++++++----------- src/bin/pg_upgrade/pg_upgrade.h | 29 +- src/bin/pg_upgrade/version.c | 288 +++------------- 3 files changed, 450 insertions(+), 462 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 64024e3b9e..cc9aa3cec2 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,14 +24,395 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(ClusterInfo *new_cluster); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = "Checking for system-defined composite types in user tables", + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + "Your installation contains system-defined composite type(s) in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = "Checking for incompatible \"line\" data type", + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + "your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. you can\n" + "drop the problem columns and restart the upgrade.\n" + "a list of the problem columns is in the file:", + .version_hook = line_type_check_applicable + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = "Checking for reg* data types in user tables", + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + "Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = "Checking for incompatible aclitem data type in user tables", + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:", + .version_hook = aclitem_type_check_applicable + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = "Checking for invalid \"unknown\" user columns", + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = unknown_type_check_applicable + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = "Checking for invalid \"sql_identifier\" user columns", + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = sql_identifier_type_check_applicable + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = "Checking for incompatible \"jsonb\" data type", + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = jsonb_9_4_check_applicable + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks *checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + n_data_types_usage_checks++; + + /* Prepare an array to store the results of checks in */ + results = pg_malloc(sizeof(bool) * n_data_types_usage_checks); + memset(results, true, sizeof(*results)); + + prep_status_progress("checking all databases"); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + continue; + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", cur_check->status); + appendPQExpBuffer(&report, "\n%s\n %s\n", + cur_check->report_text, output_path); + } + results[checknum] = false; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -100,16 +482,9 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -141,21 +516,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -164,14 +530,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1084,185 +1442,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) check_ok(); } - -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite type(s) in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"aclitem\" data type in user tables"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 3eea0139c7..4d73ed8fb7 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -328,6 +328,21 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + /* * Global variables */ @@ -450,18 +465,14 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool line_type_check_applicable(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); +bool unknown_type_check_applicable(ClusterInfo *cluster); +bool sql_identifier_type_check_applicable(ClusterInfo *cluster); +bool aclitem_type_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 403a6d7cfa..a51cb7eafa 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,69 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +line_type_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; - - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - free(base_query); - - return found; + return false; } - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) +bool +unknown_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) +bool +sql_identifier_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) + return true; + + return false; +} - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); +bool +aclitem_type_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; } /* @@ -353,41 +186,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-07-10 23:09 Nathan Bossart <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Nathan Bossart @ 2023-07-10 23:09 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> On Mon, Jul 10, 2023 at 04:43:23PM +0200, Daniel Gustafsson wrote: >> On 8 Jul 2023, at 23:43, Nathan Bossart <[email protected]> wrote: >> Since "script" will be NULL and "db_used" will be false in the first >> iteration of the loop, couldn't we move this stuff to before the loop? > > We could. It's done this way to match how all the other checks are performing > the inner loop for consistency. I think being consistent is better than micro > optimizing in non-hot codepaths even though it adds some redundancy. > > [ ... ] > >> Won't "script" always be initialized here? If I'm following this code >> correctly, I think everything except the fclose() can be removed. > > You are right that this check is superfluous. This is again an artifact of > modelling the code around how the other checks work for consistency. At least > I think that's a good characteristic of the code. I can't say I agree with this, but I'm not going to hold up the patch over it. FWIW I was looking at this more from a code simplification/readability standpoint. >> +static int n_data_types_usage_checks = 7; >> >> Can we determine this programmatically so that folks don't need to remember >> to update it? > > Fair point, I've added a counter loop to the beginning of the check function to > calculate it. + /* Gather number of checks to perform */ + while (tmp->status != NULL) + n_data_types_usage_checks++; I think we need to tmp++ somewhere here. >> In fact, I wonder if we could just add the versions directly to >> data_types_usage_checks so that we don't need the separate hook functions. > > We could, but it would be sort of contrived I think since some check <= and > some == while some check the catversion as well (and new ones may have other > variants. I think this is the least paint-ourselves-in-a-corner version, if we > feel it's needlessly complicated and no other variants are added we can revisit > this. Makes sense. -- Nathan Bossart Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-07-10 23:26 Daniel Gustafsson <[email protected]> parent: Nathan Bossart <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2023-07-10 23:26 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> > On 11 Jul 2023, at 01:09, Nathan Bossart <[email protected]> wrote: > On Mon, Jul 10, 2023 at 04:43:23PM +0200, Daniel Gustafsson wrote: >>> +static int n_data_types_usage_checks = 7; >>> >>> Can we determine this programmatically so that folks don't need to remember >>> to update it? >> >> Fair point, I've added a counter loop to the beginning of the check function to >> calculate it. > > + /* Gather number of checks to perform */ > + while (tmp->status != NULL) > + n_data_types_usage_checks++; > > I think we need to tmp++ somewhere here. Yuk, yes, will fix when caffeinated. Thanks. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-07-11 22:43 Daniel Gustafsson <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2023-07-11 22:43 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> > On 11 Jul 2023, at 01:26, Daniel Gustafsson <[email protected]> wrote: >> On 11 Jul 2023, at 01:09, Nathan Bossart <[email protected]> wrote: >> I think we need to tmp++ somewhere here. > > Yuk, yes, will fix when caffeinated. Thanks. I did have coffee before now, but only found time to actually address this now so here is a v7 with just that change and a fresh rebase. -- Daniel Gustafsson Attachments: [application/octet-stream] v7-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch (37.2K, ../../[email protected]/2-v7-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch) download | inline diff: From 4ec6774b676e0e68d486e5a10e29b148fcfe509c Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Thu, 6 Jul 2023 17:55:45 +0200 Subject: [PATCH v7] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 598 +++++++++++++++++++++----------- src/bin/pg_upgrade/pg_upgrade.h | 29 +- src/bin/pg_upgrade/version.c | 288 +++------------ 3 files changed, 453 insertions(+), 462 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 64024e3b9e..815ff58540 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,14 +24,398 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(ClusterInfo *new_cluster); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = "Checking for system-defined composite types in user tables", + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + "Your installation contains system-defined composite type(s) in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = "Checking for incompatible \"line\" data type", + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + "your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. you can\n" + "drop the problem columns and restart the upgrade.\n" + "a list of the problem columns is in the file:", + .version_hook = line_type_check_applicable + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = "Checking for reg* data types in user tables", + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + "Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = "Checking for incompatible aclitem data type in user tables", + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:", + .version_hook = aclitem_type_check_applicable + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = "Checking for invalid \"unknown\" user columns", + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = unknown_type_check_applicable + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = "Checking for invalid \"sql_identifier\" user columns", + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = sql_identifier_type_check_applicable + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = "Checking for incompatible \"jsonb\" data type", + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = jsonb_9_4_check_applicable + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks *checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc(sizeof(bool) * n_data_types_usage_checks); + memset(results, true, sizeof(*results)); + + prep_status_progress("checking all databases"); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + continue; + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", cur_check->status); + appendPQExpBuffer(&report, "\n%s\n %s\n", + cur_check->report_text, output_path); + } + results[checknum] = false; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -100,16 +485,9 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -141,21 +519,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -164,14 +533,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1084,185 +1445,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) check_ok(); } - -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite type(s) in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"aclitem\" data type in user tables"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 3eea0139c7..4d73ed8fb7 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -328,6 +328,21 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + /* * Global variables */ @@ -450,18 +465,14 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool line_type_check_applicable(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); +bool unknown_type_check_applicable(ClusterInfo *cluster); +bool sql_identifier_type_check_applicable(ClusterInfo *cluster); +bool aclitem_type_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 403a6d7cfa..a51cb7eafa 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,69 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +line_type_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; - - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - free(base_query); - - return found; + return false; } - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) +bool +unknown_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) +bool +sql_identifier_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) + return true; + + return false; +} - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); +bool +aclitem_type_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; } /* @@ -353,41 +186,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-07-11 23:36 Nathan Bossart <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Nathan Bossart @ 2023-07-11 23:36 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> On Wed, Jul 12, 2023 at 12:43:14AM +0200, Daniel Gustafsson wrote: > I did have coffee before now, but only found time to actually address this now > so here is a v7 with just that change and a fresh rebase. Thanks. I think the patch is in decent shape. -- Nathan Bossart Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-08-31 21:34 Daniel Gustafsson <[email protected]> parent: Nathan Bossart <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2023-08-31 21:34 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> > On 12 Jul 2023, at 01:36, Nathan Bossart <[email protected]> wrote: > > On Wed, Jul 12, 2023 at 12:43:14AM +0200, Daniel Gustafsson wrote: >> I did have coffee before now, but only found time to actually address this now >> so here is a v7 with just that change and a fresh rebase. > > Thanks. I think the patch is in decent shape. Due to ENOTENOUGHTIME it bitrotted a bit, so here is a v8 rebase which I really hope to close in this CF. -- Daniel Gustafsson Attachments: [application/octet-stream] v8-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch (37.1K, ../../[email protected]/2-v8-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch) download | inline diff: From 37922039d3439f88b115f57d61ced98b32bbd953 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Thu, 31 Aug 2023 23:28:34 +0200 Subject: [PATCH v8] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 598 +++++++++++++++++++++----------- src/bin/pg_upgrade/pg_upgrade.h | 29 +- src/bin/pg_upgrade/version.c | 288 +++------------ 3 files changed, 453 insertions(+), 462 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 56e313f562..b548aebae3 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,14 +24,398 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = "Checking for system-defined composite types in user tables", + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + "Your installation contains system-defined composite type(s) in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = "Checking for incompatible \"line\" data type", + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + "your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. you can\n" + "drop the problem columns and restart the upgrade.\n" + "a list of the problem columns is in the file:", + .version_hook = line_type_check_applicable + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = "Checking for reg* data types in user tables", + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + "Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = "Checking for incompatible aclitem data type in user tables", + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:", + .version_hook = aclitem_type_check_applicable + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = "Checking for invalid \"unknown\" user columns", + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = unknown_type_check_applicable + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = "Checking for invalid \"sql_identifier\" user columns", + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = sql_identifier_type_check_applicable + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = "Checking for incompatible \"jsonb\" data type", + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = jsonb_9_4_check_applicable + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks *checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc(sizeof(bool) * n_data_types_usage_checks); + memset(results, true, sizeof(*results)); + + prep_status_progress("checking all databases"); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + continue; + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", cur_check->status); + appendPQExpBuffer(&report, "\n%s\n %s\n", + cur_check->report_text, output_path); + } + results[checknum] = false; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -100,16 +485,9 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -141,21 +519,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -164,14 +533,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1084,185 +1445,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) check_ok(); } - -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"aclitem\" data type in user tables"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 7afa96716e..83fe15efca 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -328,6 +328,21 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + /* * Global variables */ @@ -450,18 +465,14 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool line_type_check_applicable(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); +bool unknown_type_check_applicable(ClusterInfo *cluster); +bool sql_identifier_type_check_applicable(ClusterInfo *cluster); +bool aclitem_type_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 403a6d7cfa..a51cb7eafa 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,69 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +line_type_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; - - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - free(base_query); - - return found; + return false; } - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) +bool +unknown_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) +bool +sql_identifier_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) + return true; + + return false; +} - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); +bool +aclitem_type_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; } /* @@ -353,41 +186,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-09-13 14:12 Peter Eisentraut <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Peter Eisentraut @ 2023-09-13 14:12 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; Nathan Bossart <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]> On 31.08.23 23:34, Daniel Gustafsson wrote: >> On 12 Jul 2023, at 01:36, Nathan Bossart <[email protected]> wrote: >> >> On Wed, Jul 12, 2023 at 12:43:14AM +0200, Daniel Gustafsson wrote: >>> I did have coffee before now, but only found time to actually address this now >>> so here is a v7 with just that change and a fresh rebase. >> >> Thanks. I think the patch is in decent shape. > > Due to ENOTENOUGHTIME it bitrotted a bit, so here is a v8 rebase which I really > hope to close in this CF. The alignment of this output looks a bit funny: ... Checking for prepared transactions ok Checking for contrib/isn with bigint-passing mismatch ok Checking for data type usage checking all databases ok Checking for presence of required libraries ok Checking database user is the install user ok ... Also, you should put gettext_noop() calls into the .status = "Checking ..." assignments and arrange to call gettext() where they are used, to maintain the translatability. ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-09-14 08:48 Daniel Gustafsson <[email protected]> parent: Peter Eisentraut <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2023-09-14 08:48 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: Nathan Bossart <[email protected]>; PostgreSQL Hackers <[email protected]> > On 13 Sep 2023, at 16:12, Peter Eisentraut <[email protected]> wrote: > The alignment of this output looks a bit funny: > > ... > Checking for prepared transactions ok > Checking for contrib/isn with bigint-passing mismatch ok > Checking for data type usage checking all databases > ok > Checking for presence of required libraries ok > Checking database user is the install user ok > ... I was using the progress reporting to indicate that it hadn't stalled for slow systems, but it's not probably not all that important really. Removed such that "ok" aligns. > Also, you should put gettext_noop() calls into the .status = "Checking ..." > assignments and arrange to call gettext() where they are used, to maintain > the translatability. Ah, yes of course. Fixed. -- Daniel Gustafsson Attachments: [application/octet-stream] v9-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch (37.2K, ../../[email protected]/2-v9-0001-pg_upgrade-run-all-data-type-checks-per-connectio.patch) download | inline diff: From 05791b7f3ebee5706940dba146fc2733d8b71669 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Thu, 14 Sep 2023 10:30:47 +0200 Subject: [PATCH v9] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Reviewed-by: Peter Eisentraut <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 596 +++++++++++++++++++++----------- src/bin/pg_upgrade/pg_upgrade.h | 29 +- src/bin/pg_upgrade/version.c | 288 +++------------ 3 files changed, 451 insertions(+), 462 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 56e313f562..5422d424a4 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,14 +24,396 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = gettext_noop("Checking for system-defined composite types in user tables"), + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + "Your installation contains system-defined composite type(s) in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = gettext_noop("Checking for incompatible \"line\" data type"), + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + "your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. you can\n" + "drop the problem columns and restart the upgrade.\n" + "a list of the problem columns is in the file:", + .version_hook = line_type_check_applicable + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = gettext_noop("Checking for reg* data types in user tables"), + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + "Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = gettext_noop("Checking for incompatible aclitem data type in user tables"), + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:", + .version_hook = aclitem_type_check_applicable + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = gettext_noop("Checking for invalid \"unknown\" user columns"), + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = unknown_type_check_applicable + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = gettext_noop("Checking for invalid \"sql_identifier\" user columns"), + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = sql_identifier_type_check_applicable + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = gettext_noop("Checking for incompatible \"jsonb\" data type"), + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = jsonb_9_4_check_applicable + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks *checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc(sizeof(bool) * n_data_types_usage_checks); + memset(results, true, sizeof(*results)); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + continue; + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", _(cur_check->status)); + appendPQExpBuffer(&report, "\n%s\n %s\n", + cur_check->report_text, output_path); + } + results[checknum] = false; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -100,16 +483,9 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -141,21 +517,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -164,14 +531,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1084,185 +1443,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) check_ok(); } - -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"aclitem\" data type in user tables"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 842f3b6cd3..313d635198 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -329,6 +329,21 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + /* * Global variables */ @@ -451,18 +466,14 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool line_type_check_applicable(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); +bool unknown_type_check_applicable(ClusterInfo *cluster); +bool sql_identifier_type_check_applicable(ClusterInfo *cluster); +bool aclitem_type_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 403a6d7cfa..a51cb7eafa 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,69 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +line_type_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; - - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - free(base_query); - - return found; + return false; } - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) +bool +unknown_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) +bool +sql_identifier_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) + return true; + + return false; +} - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); +bool +aclitem_type_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; } /* @@ -353,41 +186,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2023-10-27 13:20 Daniel Gustafsson <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2023-10-27 13:20 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: Nathan Bossart <[email protected]>; PostgreSQL Hackers <[email protected]> Attached is a v10 rebase of this patch which had undergone significant bitrot due to recent changes in the pg_upgrade check phase. This brings in the changes into the proposed structure without changes to queries, with no additional changes to the proposed functionality. Testing with a completely empty v11 cluster fresh from initdb as the old cluster shows a significant speedup (averaged over multiple runs, adjusted for outliers): patched: 53.59ms (52.78ms, 52.49ms, 55.49ms) master : 125.87ms (125.23 ms, 125.67ms, 126.67ms) Using a similarly empty cluster from master as the old cluster shows a smaller speedup, which is expected since many checks only run for older versions: patched: 33.36ms (32.82ms, 33.78ms, 33.47ms) master : 44.87ms (44.73ms, 44.90ms 44.99ms) The latter case is still pretty interesting IMO since it can speed up testing where every millisecond gained matters. -- Daniel Gustafsson Attachments: [application/octet-stream] v10-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch (41.4K, ../../[email protected]/2-v10-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch) download | inline diff: From 08a69b7220279db98462e0b96dadeba4d227eba4 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Fri, 27 Oct 2023 13:44:28 +0200 Subject: [PATCH v10] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Reviewed-by: Peter Eisentraut <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 687 ++++++++++++++++++++------------ src/bin/pg_upgrade/pg_upgrade.h | 30 +- src/bin/pg_upgrade/version.c | 295 +++----------- 3 files changed, 504 insertions(+), 508 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index fa52aa2c22..d94e2fe401 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,19 +24,441 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_removed_data_type_usage(ClusterInfo *cluster, - const char *version, - const char *datatype); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); static void check_new_cluster_logical_replication_slots(void); static void check_old_cluster_for_valid_slots(bool live_check); +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = gettext_noop("Checking for system-defined composite types in user tables"), + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + "Your installation contains system-defined composite types in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = gettext_noop("Checking for incompatible \"line\" data type"), + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + "your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. you can\n" + "drop the problem columns and restart the upgrade.\n" + "a list of the problem columns is in the file:", + .version_hook = line_type_check_applicable + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = gettext_noop("Checking for reg* data types in user tables"), + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + "Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = gettext_noop("Checking for incompatible \"aclitem\" data type"), + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:", + .version_hook = aclitem_type_check_applicable + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = gettext_noop("Checking for invalid \"unknown\" user columns"), + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = unknown_type_check_applicable + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = gettext_noop("Checking for invalid \"sql_identifier\" user columns"), + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = sql_identifier_type_check_applicable + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = gettext_noop("Checking for incompatible \"jsonb\" data type in user tables"), + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = jsonb_9_4_check_applicable + }, + + /* + * PG 12 removed types abstime, reltime, tinterval. + */ + { + .status = gettext_noop("Checking for removed \"abstime\" data type in user tables"), + .report_filename = "tables_using_abstime.txt", + .base_query = + "SELECT 'pg_catalog.abstime'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"abstime\" data type in user tables.\n" + "The \"abstime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:\n", + .version_hook = removed_data_types_check_applicable + }, + { + .status = gettext_noop("Checking for removed \"reltime\" data type in user tables"), + .report_filename = "tables_using_reltime.txt", + .base_query = + "SELECT 'pg_catalog.reltime'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"reltime\" data type in user tables.\n" + "The \"reltime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:\n", + .version_hook = removed_data_types_check_applicable + }, + { + .status = gettext_noop("Checking for removed \"tinterval\" data type in user tables"), + .report_filename = "tables_using_tinterval.txt", + .base_query = + "SELECT 'pg_catalog.tinterval'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"tinterval\" data type in user tables.\n" + "The \"tinterval\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:\n", + .version_hook = removed_data_types_check_applicable + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks *checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc(sizeof(bool) * n_data_types_usage_checks); + memset(results, true, sizeof(*results)); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + continue; + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", _(cur_check->status)); + appendPQExpBuffer(&report, "\n%s\n %s\n", + cur_check->report_text, output_path); + } + results[checknum] = false; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -108,8 +531,6 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); /* @@ -119,22 +540,7 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1700) check_old_cluster_for_valid_slots(live_check); - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); - - /* - * PG 12 removed types abstime, reltime, tinterval. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - { - check_for_removed_data_type_usage(&old_cluster, "12", "abstime"); - check_for_removed_data_type_usage(&old_cluster, "12", "reltime"); - check_for_removed_data_type_usage(&old_cluster, "12", "tinterval"); - } + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -166,21 +572,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -189,14 +586,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1112,220 +1501,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) } -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"%s\" data type in user tables", - "aclitem"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_removed_data_type_usage - * - * Check for in-core data types that have been removed. Callers know - * the exact list. - */ -static void -check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, - const char *datatype) -{ - char output_path[MAXPGPATH]; - char typename[NAMEDATALEN]; - - prep_status("Checking for removed \"%s\" data type in user tables", - datatype); - - snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", - datatype); - snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); - - if (check_for_data_type_usage(cluster, typename, output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" - "The \"%s\" type has been removed in PostgreSQL version %s,\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns, or change them to another data type, and restart\n" - "the upgrade. A list of the problem columns is in the file:\n" - " %s", datatype, datatype, version, output_path); - } - else - check_ok(); -} - - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index a710f325de..f949d5de9b 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -348,6 +348,21 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + /* * Global variables */ @@ -471,18 +486,15 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool line_type_check_applicable(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); +bool unknown_type_check_applicable(ClusterInfo *cluster); +bool sql_identifier_type_check_applicable(ClusterInfo *cluster); +bool aclitem_type_check_applicable(ClusterInfo *cluster); +bool removed_data_types_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 403a6d7cfa..4382957b59 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,80 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +line_type_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } - - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - return found; + return false; } - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) +bool +unknown_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) +bool +sql_identifier_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1100) + return true; + + return false; +} - prep_status("Checking for invalid \"unknown\" user columns"); +bool +aclitem_type_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; +} - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); +bool +removed_data_types_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 12 removed abstime, reltime and tinterval */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1100) + return true; - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + return false; } /* @@ -353,41 +197,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-01-27 03:40 vignesh C <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: vignesh C @ 2024-01-27 03:40 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; Nathan Bossart <[email protected]>; PostgreSQL Hackers <[email protected]> On Fri, 27 Oct 2023 at 18:50, Daniel Gustafsson <[email protected]> wrote: > > Attached is a v10 rebase of this patch which had undergone significant bitrot > due to recent changes in the pg_upgrade check phase. This brings in the > changes into the proposed structure without changes to queries, with no > additional changes to the proposed functionality. > > Testing with a completely empty v11 cluster fresh from initdb as the old > cluster shows a significant speedup (averaged over multiple runs, adjusted for > outliers): > > patched: 53.59ms (52.78ms, 52.49ms, 55.49ms) > master : 125.87ms (125.23 ms, 125.67ms, 126.67ms) > > Using a similarly empty cluster from master as the old cluster shows a smaller > speedup, which is expected since many checks only run for older versions: > > patched: 33.36ms (32.82ms, 33.78ms, 33.47ms) > master : 44.87ms (44.73ms, 44.90ms 44.99ms) > > The latter case is still pretty interesting IMO since it can speed up testing > where every millisecond gained matters. CFBot shows that the patch does not apply anymore as in [1]: === Applying patches on top of PostgreSQL commit ID 55627ba2d334ce98e1f5916354c46472d414bda6 === === applying patch ./v10-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch patching file src/bin/pg_upgrade/check.c Hunk #2 FAILED at 24. ... 1 out of 7 hunks FAILED -- saving rejects to file src/bin/pg_upgrade/check.c.rej Please post an updated version for the same. [1] - http://cfbot.cputube.org/patch_46_4200.log Regards, Vignesh ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-01 18:48 vignesh C <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: vignesh C @ 2024-02-01 18:48 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: Peter Eisentraut <[email protected]>; Nathan Bossart <[email protected]>; PostgreSQL Hackers <[email protected]> On Sat, 27 Jan 2024 at 09:10, vignesh C <[email protected]> wrote: > > On Fri, 27 Oct 2023 at 18:50, Daniel Gustafsson <[email protected]> wrote: > > > > Attached is a v10 rebase of this patch which had undergone significant bitrot > > due to recent changes in the pg_upgrade check phase. This brings in the > > changes into the proposed structure without changes to queries, with no > > additional changes to the proposed functionality. > > > > Testing with a completely empty v11 cluster fresh from initdb as the old > > cluster shows a significant speedup (averaged over multiple runs, adjusted for > > outliers): > > > > patched: 53.59ms (52.78ms, 52.49ms, 55.49ms) > > master : 125.87ms (125.23 ms, 125.67ms, 126.67ms) > > > > Using a similarly empty cluster from master as the old cluster shows a smaller > > speedup, which is expected since many checks only run for older versions: > > > > patched: 33.36ms (32.82ms, 33.78ms, 33.47ms) > > master : 44.87ms (44.73ms, 44.90ms 44.99ms) > > > > The latter case is still pretty interesting IMO since it can speed up testing > > where every millisecond gained matters. > > CFBot shows that the patch does not apply anymore as in [1]: > === Applying patches on top of PostgreSQL commit ID > 55627ba2d334ce98e1f5916354c46472d414bda6 === > === applying patch > ./v10-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch > patching file src/bin/pg_upgrade/check.c > Hunk #2 FAILED at 24. > ... > 1 out of 7 hunks FAILED -- saving rejects to file src/bin/pg_upgrade/check.c.rej > > Please post an updated version for the same. With no update to the thread and the patch still not applying I'm marking this as returned with feedback. Please feel free to resubmit to the next CF when there is a new version of the patch. Regards, Vignesh ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-06 16:32 Nathan Bossart <[email protected]> parent: vignesh C <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Nathan Bossart @ 2024-02-06 16:32 UTC (permalink / raw) To: vignesh C <[email protected]>; +Cc: Daniel Gustafsson <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]> On Fri, Feb 02, 2024 at 12:18:25AM +0530, vignesh C wrote: > With no update to the thread and the patch still not applying I'm > marking this as returned with feedback. Please feel free to resubmit > to the next CF when there is a new version of the patch. IMHO this patch is worth trying to get into v17. I'd be happy to take it forward if Daniel does not intend to work on it. -- Nathan Bossart Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-06 16:47 Daniel Gustafsson <[email protected]> parent: Nathan Bossart <[email protected]> 0 siblings, 2 replies; 25+ messages in thread From: Daniel Gustafsson @ 2024-02-06 16:47 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: vignesh C <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]> > On 6 Feb 2024, at 17:32, Nathan Bossart <[email protected]> wrote: > > On Fri, Feb 02, 2024 at 12:18:25AM +0530, vignesh C wrote: >> With no update to the thread and the patch still not applying I'm >> marking this as returned with feedback. Please feel free to resubmit >> to the next CF when there is a new version of the patch. > > IMHO this patch is worth trying to get into v17. I'd be happy to take it > forward if Daniel does not intend to work on it. I actually had the same thought yesterday and spent some time polishing and rebasing it. I'll post an updated rebase shortly with the hopes of getting it committed this week. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-06 16:55 Nathan Bossart <[email protected]> parent: Daniel Gustafsson <[email protected]> 1 sibling, 0 replies; 25+ messages in thread From: Nathan Bossart @ 2024-02-06 16:55 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: vignesh C <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]> On Tue, Feb 06, 2024 at 05:47:56PM +0100, Daniel Gustafsson wrote: >> On 6 Feb 2024, at 17:32, Nathan Bossart <[email protected]> wrote: >> IMHO this patch is worth trying to get into v17. I'd be happy to take it >> forward if Daniel does not intend to work on it. > > I actually had the same thought yesterday and spent some time polishing and > rebasing it. I'll post an updated rebase shortly with the hopes of getting it > committed this week. Oh, awesome. Thanks! -- Nathan Bossart Amazon Web Services: https://aws.amazon.com ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-07 13:25 Daniel Gustafsson <[email protected]> parent: Daniel Gustafsson <[email protected]> 1 sibling, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2024-02-07 13:25 UTC (permalink / raw) To: Nathan Bossart <[email protected]>; +Cc: vignesh C <[email protected]>; Peter Eisentraut <[email protected]>; PostgreSQL Hackers <[email protected]> > On 6 Feb 2024, at 17:47, Daniel Gustafsson <[email protected]> wrote: > >> On 6 Feb 2024, at 17:32, Nathan Bossart <[email protected]> wrote: >> >> On Fri, Feb 02, 2024 at 12:18:25AM +0530, vignesh C wrote: >>> With no update to the thread and the patch still not applying I'm >>> marking this as returned with feedback. Please feel free to resubmit >>> to the next CF when there is a new version of the patch. >> >> IMHO this patch is worth trying to get into v17. I'd be happy to take it >> forward if Daniel does not intend to work on it. > > I actually had the same thought yesterday and spent some time polishing and > rebasing it. I'll post an updated rebase shortly with the hopes of getting it > committed this week. Attached is a v11 rebased over HEAD with some very minor tweaks. Unless there are objections I plan to go ahead with this version this week. -- Daniel Gustafsson Attachments: [application/octet-stream] v11-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch (41.4K, ../../[email protected]/2-v11-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch) download | inline diff: From 2909537daddd43231a13f73f13adec91974a4f90 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Wed, 7 Feb 2024 13:36:46 +0100 Subject: [PATCH v11] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Reviewed-by: Peter Eisentraut <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 686 ++++++++++++++++++++------------ src/bin/pg_upgrade/pg_upgrade.h | 30 +- src/bin/pg_upgrade/version.c | 296 +++----------- 3 files changed, 504 insertions(+), 508 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index e36a7328bf..566b49dbba 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,13 +24,6 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_removed_data_type_usage(ClusterInfo *cluster, - const char *version, - const char *datatype); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); @@ -38,6 +32,434 @@ static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(bool live_check); static void check_old_cluster_subscription_state(void); +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = gettext_noop("Checking for system-defined composite types in user tables"), + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + "Your installation contains system-defined composite types in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = gettext_noop("Checking for incompatible \"line\" data type"), + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + "your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. you can\n" + "drop the problem columns and restart the upgrade.\n" + "a list of the problem columns is in the file:", + .version_hook = line_type_check_applicable + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = gettext_noop("Checking for reg* data types in user tables"), + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + "Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = gettext_noop("Checking for incompatible \"aclitem\" data type"), + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:", + .version_hook = aclitem_type_check_applicable + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = gettext_noop("Checking for invalid \"unknown\" user columns"), + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = unknown_type_check_applicable + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = gettext_noop("Checking for invalid \"sql_identifier\" user columns"), + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = sql_identifier_type_check_applicable + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = gettext_noop("Checking for incompatible \"jsonb\" data type in user tables"), + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:", + .version_hook = jsonb_9_4_check_applicable + }, + + /* + * PG 12 removed types abstime, reltime, tinterval. + */ + { + .status = gettext_noop("Checking for removed \"abstime\" data type in user tables"), + .report_filename = "tables_using_abstime.txt", + .base_query = + "SELECT 'pg_catalog.abstime'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"abstime\" data type in user tables.\n" + "The \"abstime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:\n", + .version_hook = removed_data_types_check_applicable + }, + { + .status = gettext_noop("Checking for removed \"reltime\" data type in user tables"), + .report_filename = "tables_using_reltime.txt", + .base_query = + "SELECT 'pg_catalog.reltime'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"reltime\" data type in user tables.\n" + "The \"reltime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:\n", + .version_hook = removed_data_types_check_applicable + }, + { + .status = gettext_noop("Checking for removed \"tinterval\" data type in user tables"), + .report_filename = "tables_using_tinterval.txt", + .base_query = + "SELECT 'pg_catalog.tinterval'::pg_catalog.regtype AS oid", + .report_text = + "Your installation contains the \"tinterval\" data type in user tables.\n" + "The \"tinterval\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:\n", + .version_hook = removed_data_types_check_applicable + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks * checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc0(sizeof(bool) * n_data_types_usage_checks); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + continue; + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (!results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", _(cur_check->status)); + appendPQExpBuffer(&report, "\n%s\n %s\n", + cur_check->report_text, output_path); + } + results[checknum] = true; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -110,8 +532,6 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1700) @@ -129,22 +549,7 @@ check_and_dump_old_cluster(bool live_check) check_old_cluster_subscription_state(); } - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); - - /* - * PG 12 removed types abstime, reltime, tinterval. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - { - check_for_removed_data_type_usage(&old_cluster, "12", "abstime"); - check_for_removed_data_type_usage(&old_cluster, "12", "reltime"); - check_for_removed_data_type_usage(&old_cluster, "12", "tinterval"); - } + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -176,21 +581,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -199,14 +595,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1124,220 +1512,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) } -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"%s\" data type in user tables", - "aclitem"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_removed_data_type_usage - * - * Check for in-core data types that have been removed. Callers know - * the exact list. - */ -static void -check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, - const char *datatype) -{ - char output_path[MAXPGPATH]; - char typename[NAMEDATALEN]; - - prep_status("Checking for removed \"%s\" data type in user tables", - datatype); - - snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", - datatype); - snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); - - if (check_for_data_type_usage(cluster, typename, output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" - "The \"%s\" type has been removed in PostgreSQL version %s,\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns, or change them to another data type, and restart\n" - "the upgrade. A list of the problem columns is in the file:\n" - " %s", datatype, datatype, version, output_path); - } - else - check_ok(); -} - - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index d9a848cbfd..23b665527f 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -351,6 +351,21 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + /* * Global variables */ @@ -475,18 +490,15 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool line_type_check_applicable(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); +bool unknown_type_check_applicable(ClusterInfo *cluster); +bool sql_identifier_type_check_applicable(ClusterInfo *cluster); +bool aclitem_type_check_applicable(ClusterInfo *cluster); +bool removed_data_types_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 13b2c0f012..2e060782ec 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,81 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +line_type_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } - - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - return found; + return false; } - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) +bool +unknown_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) +bool +sql_identifier_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1100) + return true; + + return false; +} - prep_status("Checking for invalid \"unknown\" user columns"); +bool +aclitem_type_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; +} - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); +bool +removed_data_types_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 12 removed abstime, reltime and tinterval + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1100) + return true; - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + return false; } /* @@ -353,41 +198,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-08 10:55 Peter Eisentraut <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Peter Eisentraut @ 2024-02-08 10:55 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; Nathan Bossart <[email protected]>; +Cc: vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]> On 07.02.24 14:25, Daniel Gustafsson wrote: >> On 6 Feb 2024, at 17:47, Daniel Gustafsson <[email protected]> wrote: >> >>> On 6 Feb 2024, at 17:32, Nathan Bossart <[email protected]> wrote: >>> >>> On Fri, Feb 02, 2024 at 12:18:25AM +0530, vignesh C wrote: >>>> With no update to the thread and the patch still not applying I'm >>>> marking this as returned with feedback. Please feel free to resubmit >>>> to the next CF when there is a new version of the patch. >>> >>> IMHO this patch is worth trying to get into v17. I'd be happy to take it >>> forward if Daniel does not intend to work on it. >> >> I actually had the same thought yesterday and spent some time polishing and >> rebasing it. I'll post an updated rebase shortly with the hopes of getting it >> committed this week. > > Attached is a v11 rebased over HEAD with some very minor tweaks. Unless there > are objections I plan to go ahead with this version this week. A few more quick comments: I think the .report_text assignments also need a gettext_noop(), like the .status assignments. The type DataTypesUsageChecks is only used in check.c, so doesn't need to be in pg_upgrade.h. Idea for further improvement: Might be nice if the DataTypesUsageVersionCheck struct also included the applicable version information, so the additional checks in version.c would no longer be necessary. ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-08 14:16 Daniel Gustafsson <[email protected]> parent: Peter Eisentraut <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2024-02-08 14:16 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: Nathan Bossart <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]> > On 8 Feb 2024, at 11:55, Peter Eisentraut <[email protected]> wrote: > A few more quick comments: Thanks for reviewing! > I think the .report_text assignments also need a gettext_noop(), like the .status assignments. Done in the attached. > The type DataTypesUsageChecks is only used in check.c, so doesn't need to be in pg_upgrade.h. Fixed. > Idea for further improvement: Might be nice if the DataTypesUsageVersionCheck struct also included the applicable version information, so the additional checks in version.c would no longer be necessary. I tried various variants of this when writing it, but since the checks aren't just checking version but also include catalog version checks it became messy. One option could perhaps be to include a version number for <= comparison, and if set to zero a function pointer to a version check function must be provided? That would handle the simple cases in a single place without messy logic, and leave the more convoluted checks with a special case function. -- Daniel Gustafsson Attachments: [application/octet-stream] v12-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch (41.6K, ../../[email protected]/2-v12-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch) download | inline diff: From c02fdf0e6dc29f53fef42dcead85c2079be597f6 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Wed, 7 Feb 2024 13:36:46 +0100 Subject: [PATCH v12] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Reviewed-by: Peter Eisentraut <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 698 ++++++++++++++++++++------------ src/bin/pg_upgrade/pg_upgrade.h | 18 +- src/bin/pg_upgrade/version.c | 296 +++----------- 3 files changed, 504 insertions(+), 508 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index e36a7328bf..1ac8587059 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,13 +24,6 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_removed_data_type_usage(ClusterInfo *cluster, - const char *version, - const char *datatype); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); @@ -38,6 +32,446 @@ static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(bool live_check); static void check_old_cluster_subscription_state(void); +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = gettext_noop("Checking for system-defined composite types in user tables"), + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + gettext_noop("Your installation contains system-defined composite types in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .version_hook = NULL + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = gettext_noop("Checking for incompatible \"line\" data type"), + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .version_hook = line_type_check_applicable + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = gettext_noop("Checking for reg* data types in user tables"), + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + gettext_noop("Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .version_hook = NULL + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = gettext_noop("Checking for incompatible \"aclitem\" data type"), + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:"), + .version_hook = aclitem_type_check_applicable + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = gettext_noop("Checking for invalid \"unknown\" user columns"), + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .version_hook = unknown_type_check_applicable + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = gettext_noop("Checking for invalid \"sql_identifier\" user columns"), + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .version_hook = sql_identifier_type_check_applicable + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = gettext_noop("Checking for incompatible \"jsonb\" data type in user tables"), + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .version_hook = jsonb_9_4_check_applicable + }, + + /* + * PG 12 removed types abstime, reltime, tinterval. + */ + { + .status = gettext_noop("Checking for removed \"abstime\" data type in user tables"), + .report_filename = "tables_using_abstime.txt", + .base_query = + "SELECT 'pg_catalog.abstime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"abstime\" data type in user tables.\n" + "The \"abstime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:"), + .version_hook = removed_data_types_check_applicable + }, + { + .status = gettext_noop("Checking for removed \"reltime\" data type in user tables"), + .report_filename = "tables_using_reltime.txt", + .base_query = + "SELECT 'pg_catalog.reltime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"reltime\" data type in user tables.\n" + "The \"reltime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:"), + .version_hook = removed_data_types_check_applicable + }, + { + .status = gettext_noop("Checking for removed \"tinterval\" data type in user tables"), + .report_filename = "tables_using_tinterval.txt", + .base_query = + "SELECT 'pg_catalog.tinterval'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"tinterval\" data type in user tables.\n" + "The \"tinterval\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:"), + .version_hook = removed_data_types_check_applicable + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks * checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc0(sizeof(bool) * n_data_types_usage_checks); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (cur_check->version_hook && !cur_check->version_hook(cluster)) + continue; + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (!results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", _(cur_check->status)); + appendPQExpBuffer(&report, "\n%s\n %s\n", + _(cur_check->report_text), output_path); + } + results[checknum] = true; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -110,8 +544,6 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1700) @@ -129,22 +561,7 @@ check_and_dump_old_cluster(bool live_check) check_old_cluster_subscription_state(); } - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); - - /* - * PG 12 removed types abstime, reltime, tinterval. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - { - check_for_removed_data_type_usage(&old_cluster, "12", "abstime"); - check_for_removed_data_type_usage(&old_cluster, "12", "reltime"); - check_for_removed_data_type_usage(&old_cluster, "12", "tinterval"); - } + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -176,21 +593,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -199,14 +607,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1124,220 +1524,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) } -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"%s\" data type in user tables", - "aclitem"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_removed_data_type_usage - * - * Check for in-core data types that have been removed. Callers know - * the exact list. - */ -static void -check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, - const char *datatype) -{ - char output_path[MAXPGPATH]; - char typename[NAMEDATALEN]; - - prep_status("Checking for removed \"%s\" data type in user tables", - datatype); - - snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", - datatype); - snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); - - if (check_for_data_type_usage(cluster, typename, output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" - "The \"%s\" type has been removed in PostgreSQL version %s,\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns, or change them to another data type, and restart\n" - "the upgrade. A list of the problem columns is in the file:\n" - " %s", datatype, datatype, version, output_path); - } - else - check_ok(); -} - - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index d9a848cbfd..494335ea93 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -351,6 +351,9 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + /* * Global variables */ @@ -475,18 +478,15 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool line_type_check_applicable(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); +bool unknown_type_check_applicable(ClusterInfo *cluster); +bool sql_identifier_type_check_applicable(ClusterInfo *cluster); +bool aclitem_type_check_applicable(ClusterInfo *cluster); +bool removed_data_types_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 13b2c0f012..2e060782ec 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,81 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +line_type_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; + /* Pre-PG 9.4 had a different 'line' data type internal format */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 903) + return true; - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } - - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; + return false; } -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - return found; + return false; } - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) +bool +unknown_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + /* Pre-PG 10 allowed tables with 'unknown' type columns */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + return true; + return false; } - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) +bool +sql_identifier_type_check_applicable(ClusterInfo *cluster) { - char output_path[MAXPGPATH]; + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1100) + return true; + + return false; +} - prep_status("Checking for invalid \"unknown\" user columns"); +bool +aclitem_type_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1500) + return true; + + return false; +} - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); +bool +removed_data_types_check_applicable(ClusterInfo *cluster) +{ + /* + * PG 12 removed abstime, reltime and tinterval + */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 1100) + return true; - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + return false; } /* @@ -353,41 +198,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-08 23:04 Daniel Gustafsson <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2024-02-08 23:04 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: Nathan Bossart <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]> > On 8 Feb 2024, at 15:16, Daniel Gustafsson <[email protected]> wrote: > One option could perhaps be to include a version number for <= comparison, and > if set to zero a function pointer to a version check function must be provided? > That would handle the simple cases in a single place without messy logic, and > leave the more convoluted checks with a special case function. The attached is a draft version of this approach, each check can define to run for all versions, set a threshold version for which it runs or define a callback which implements a more complicated check. -- Daniel Gustafsson Attachments: [application/octet-stream] v13-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch (40.3K, ../../[email protected]/2-v13-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch) download | inline diff: From 7e8ec96cf4ff291595482ae2226e82cbdf16662e Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Wed, 7 Feb 2024 13:36:46 +0100 Subject: [PATCH v13] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Reviewed-by: Peter Eisentraut <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 715 ++++++++++++++++++++------------ src/bin/pg_upgrade/pg_upgrade.h | 13 +- src/bin/pg_upgrade/version.c | 266 +----------- 3 files changed, 472 insertions(+), 522 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index e36a7328bf..663e270f30 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,13 +24,6 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_removed_data_type_usage(ClusterInfo *cluster, - const char *version, - const char *datatype); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); @@ -38,6 +32,463 @@ static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(bool live_check); static void check_old_cluster_subscription_state(void); +#define MANUAL_CHECK 0 +#define ALL_VERSIONS -1 + +/* + * DataTypesUsageChecks + */ +typedef struct +{ + const char *status; /* status line to print to the user */ + const char *report_filename; /* filename to store report to */ + const char *base_query; /* Query to extract the oid of the datatype */ + const char *report_text; /* Text to store to report in case of error */ + int threshold_version; + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + +/* + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and a function pointer for determining if the check should be executed + * for the current version. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = gettext_noop("Checking for system-defined composite types in user tables"), + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + gettext_noop("Your installation contains system-defined composite types in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .threshold_version = ALL_VERSIONS + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = gettext_noop("Checking for incompatible \"line\" data type"), + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .threshold_version = 903 + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = gettext_noop("Checking for reg* data types in user tables"), + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + gettext_noop("Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .threshold_version = ALL_VERSIONS + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = gettext_noop("Checking for incompatible \"aclitem\" data type"), + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade. A list of the problem\n" + "columns is in the file:"), + .threshold_version = 1500 + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = gettext_noop("Checking for invalid \"unknown\" user columns"), + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .threshold_version = 906 + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = gettext_noop("Checking for invalid \"sql_identifier\" user columns"), + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .threshold_version = 1100 + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = gettext_noop("Checking for incompatible \"jsonb\" data type in user tables"), + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n" + "A list of the problem columns is in the file:"), + .threshold_version = MANUAL_CHECK, + .version_hook = jsonb_9_4_check_applicable + }, + + /* + * PG 12 removed types abstime, reltime, tinterval. + */ + { + .status = gettext_noop("Checking for removed \"abstime\" data type in user tables"), + .report_filename = "tables_using_abstime.txt", + .base_query = + "SELECT 'pg_catalog.abstime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"abstime\" data type in user tables.\n" + "The \"abstime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:"), + .threshold_version = 1100 + }, + { + .status = gettext_noop("Checking for removed \"reltime\" data type in user tables"), + .report_filename = "tables_using_reltime.txt", + .base_query = + "SELECT 'pg_catalog.reltime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"reltime\" data type in user tables.\n" + "The \"reltime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:"), + .threshold_version = 1100 + }, + { + .status = gettext_noop("Checking for removed \"tinterval\" data type in user tables"), + .report_filename = "tables_using_tinterval.txt", + .base_query = + "SELECT 'pg_catalog.tinterval'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"tinterval\" data type in user tables.\n" + "The \"tinterval\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade. A list of the problem columns is in the file:"), + .threshold_version = 1100 + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, 0, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks * checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc0(sizeof(bool) * n_data_types_usage_checks); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + if (cur_check->threshold_version == MANUAL_CHECK) + { + Assert(cur_check->version_hook); + + /* + * Make sure that the check applies to the current cluster version + * and skip if not. If no check hook has been defined we run the + * check for all versions. + */ + if (!cur_check->version_hook(cluster)) + continue; + } + else if (cur_check->threshold_version != ALL_VERSIONS) + { + if (GET_MAJOR_VERSION(cluster->major_version) > cur_check->threshold_version) + continue; + } + else + Assert(cur_check->threshold_version == ALL_VERSIONS); + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (!results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", _(cur_check->status)); + appendPQExpBuffer(&report, "\n%s\n %s\n", + _(cur_check->report_text), output_path); + } + results[checknum] = true; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -110,8 +561,6 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1700) @@ -129,22 +578,7 @@ check_and_dump_old_cluster(bool live_check) check_old_cluster_subscription_state(); } - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); - - /* - * PG 12 removed types abstime, reltime, tinterval. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - { - check_for_removed_data_type_usage(&old_cluster, "12", "abstime"); - check_for_removed_data_type_usage(&old_cluster, "12", "reltime"); - check_for_removed_data_type_usage(&old_cluster, "12", "tinterval"); - } + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -176,21 +610,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -199,14 +624,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1124,220 +1541,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) } -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"%s\" data type in user tables", - "aclitem"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_removed_data_type_usage - * - * Check for in-core data types that have been removed. Callers know - * the exact list. - */ -static void -check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, - const char *datatype) -{ - char output_path[MAXPGPATH]; - char typename[NAMEDATALEN]; - - prep_status("Checking for removed \"%s\" data type in user tables", - datatype); - - snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", - datatype); - snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); - - if (check_for_data_type_usage(cluster, typename, output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" - "The \"%s\" type has been removed in PostgreSQL version %s,\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns, or change them to another data type, and restart\n" - "the upgrade. A list of the problem columns is in the file:\n" - " %s", datatype, datatype, version, output_path); - } - else - check_ok(); -} - - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index d9a848cbfd..7daa67f809 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -351,6 +351,9 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + /* * Global variables */ @@ -475,18 +478,10 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 13b2c0f012..e3c7b4109c 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,23 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - -/* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. - */ -bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) -{ - bool found = false; - FILE *script = NULL; - int dbnum; - - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } - - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; -} - /* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - return found; -} - - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + return false; } /* @@ -353,41 +140,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-02-09 09:33 Daniel Gustafsson <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2024-02-09 09:33 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: Nathan Bossart <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]> > On 9 Feb 2024, at 00:04, Daniel Gustafsson <[email protected]> wrote: > >> On 8 Feb 2024, at 15:16, Daniel Gustafsson <[email protected]> wrote: > >> One option could perhaps be to include a version number for <= comparison, and >> if set to zero a function pointer to a version check function must be provided? >> That would handle the simple cases in a single place without messy logic, and >> leave the more convoluted checks with a special case function. > > The attached is a draft version of this approach, each check can define to run > for all versions, set a threshold version for which it runs or define a > callback which implements a more complicated check. And again pgindented and with documentation on the struct members to make it easy to add new checks. A repetitive part of the report text was also moved to a single place. -- Daniel Gustafsson Attachments: [application/octet-stream] v14-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch (42.1K, ../../[email protected]/2-v14-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch) download | inline diff: From 2d9ec92dd3af34ab240364445a404fef2d46caa1 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Wed, 7 Feb 2024 13:36:46 +0100 Subject: [PATCH v14] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On cluster which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize connection setup/teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Reviewed-by: Peter Eisentraut <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 750 +++++++++++++++++++++----------- src/bin/pg_upgrade/pg_upgrade.h | 13 +- src/bin/pg_upgrade/version.c | 266 +---------- 3 files changed, 507 insertions(+), 522 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index e36a7328bf..6d640d7bb0 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,13 +24,6 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_removed_data_type_usage(ClusterInfo *cluster, - const char *version, - const char *datatype); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); @@ -38,6 +32,498 @@ static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(bool live_check); static void check_old_cluster_subscription_state(void); +/* + * DataTypesUsageChecks - definitions of data type checks for the old cluster + * in order to determine if an upgrade can be performed. See the comment on + * data_types_usage_checks below for a more detailed description. + */ +typedef struct +{ + /* Status line to print to the user */ + const char *status; + /* Filename to store report to */ + const char *report_filename; + /* Query to extract the oid of the datatype */ + const char *base_query; + /* Text to store to report in case of error */ + const char *report_text; + /* The latest version where the check applies */ + int threshold_version; + /* A function pointer for determining if the check applies */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + +/* + * Special values for threshold_version for indicating that a check applies to + * all versions, or that a custom function needs to be invoked to determine + * if the check applies. + */ +#define MANUAL_CHECK 1 +#define ALL_VERSIONS -1 + +/*-- + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and functionality for deciding if the check is applicable to the version + * of the old cluster. The struct members are described in detail below: + * + * status A oneline string which can be printed to the user to + * inform about progress. Should not end with newline. + * report_filename The filename in which the list of problems detected by + * the check will be printed. + * base_query A query which extracts the Oid of the datatype checked + * for. + * report_text The text which will be printed to the user to explain + * what the check did, and why it failed. The text should + * end with a newline, and does not need to refer to the + * report_filename as that is automatically appended to + * the report with the path to the log folder. + * threshold_version The major version of PostgreSQL for which to run the + * check. Iff the old cluster is less than, or equal to, + * the threshold version then the check will be executed. + * If the old version is greater than the threshold then + * the check is skipped. If the threshold_version is set + * to ALL_VERSIONS then it will be run unconditionally, + * if set to MANUAL_CHECK then the version_hook function + * will be executed in order to determine whether or not + * to run. + * version_hook A function pointer to a version check function of type + * DataTypesUsageVersionCheck which is used to determine + * if the check is applicable to the old cluster. If the + * version_hook returns true then the check will be run, + * else it will be skipped. The function will only be + * executed iff threshold_version is set to MANUAL_CHECK. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = gettext_noop("Checking for system-defined composite types in user tables"), + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + gettext_noop("Your installation contains system-defined composite types in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade.\n"), + .threshold_version = ALL_VERSIONS + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = gettext_noop("Checking for incompatible \"line\" data type"), + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n"), + .threshold_version = 903 + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = gettext_noop("Checking for reg* data types in user tables"), + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + gettext_noop("Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n"), + .threshold_version = ALL_VERSIONS + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = gettext_noop("Checking for incompatible \"aclitem\" data type"), + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade.\n"), + .threshold_version = 1500 + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = gettext_noop("Checking for invalid \"unknown\" user columns"), + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this cluster\n" + "cannot currently be upgraded. You can drop the problem columns\n" + "and restart the upgrade.\n"), + .threshold_version = 906 + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = gettext_noop("Checking for invalid \"sql_identifier\" user columns"), + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can drop the problem\n" + "columns and restart the upgrade.\n"), + .threshold_version = 1100 + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = gettext_noop("Checking for incompatible \"jsonb\" data type in user tables"), + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can drop the problem \n" + "columns and restart the upgrade.\n"), + .threshold_version = MANUAL_CHECK, + .version_hook = jsonb_9_4_check_applicable + }, + + /* + * PG 12 removed types abstime, reltime, tinterval. + */ + { + .status = gettext_noop("Checking for removed \"abstime\" data type in user tables"), + .report_filename = "tables_using_abstime.txt", + .base_query = + "SELECT 'pg_catalog.abstime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"abstime\" data type in user tables.\n" + "The \"abstime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade.\n"), + .threshold_version = 1100 + }, + { + .status = gettext_noop("Checking for removed \"reltime\" data type in user tables"), + .report_filename = "tables_using_reltime.txt", + .base_query = + "SELECT 'pg_catalog.reltime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"reltime\" data type in user tables.\n" + "The \"reltime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade.\n"), + .threshold_version = 1100 + }, + { + .status = gettext_noop("Checking for removed \"tinterval\" data type in user tables"), + .report_filename = "tables_using_tinterval.txt", + .base_query = + "SELECT 'pg_catalog.tinterval'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"tinterval\" data type in user tables.\n" + "The \"tinterval\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade.\n"), + .threshold_version = 1100 + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, 0, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks * checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc0(sizeof(bool) * n_data_types_usage_checks); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + if (cur_check->threshold_version == MANUAL_CHECK) + { + Assert(cur_check->version_hook); + + /* + * Make sure that the check applies to the current cluster + * version and skip if not. If no check hook has been defined + * we run the check for all versions. + */ + if (!cur_check->version_hook(cluster)) + continue; + } + else if (cur_check->threshold_version != ALL_VERSIONS) + { + if (GET_MAJOR_VERSION(cluster->major_version) > cur_check->threshold_version) + continue; + } + else + Assert(cur_check->threshold_version == ALL_VERSIONS); + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (!results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", _(cur_check->status)); + appendPQExpBuffer(&report, "\n%s\n%s %s\n", + _(cur_check->report_text), + _("A list of the problem columns is in the file:"), + output_path); + } + results[checknum] = true; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %s", + output_path, + strerror(errno)); + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -110,8 +596,6 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1700) @@ -129,22 +613,7 @@ check_and_dump_old_cluster(bool live_check) check_old_cluster_subscription_state(); } - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); - - /* - * PG 12 removed types abstime, reltime, tinterval. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - { - check_for_removed_data_type_usage(&old_cluster, "12", "abstime"); - check_for_removed_data_type_usage(&old_cluster, "12", "reltime"); - check_for_removed_data_type_usage(&old_cluster, "12", "tinterval"); - } + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -176,21 +645,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -199,14 +659,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1124,220 +1576,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) } -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"%s\" data type in user tables", - "aclitem"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_removed_data_type_usage - * - * Check for in-core data types that have been removed. Callers know - * the exact list. - */ -static void -check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, - const char *datatype) -{ - char output_path[MAXPGPATH]; - char typename[NAMEDATALEN]; - - prep_status("Checking for removed \"%s\" data type in user tables", - datatype); - - snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", - datatype); - snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); - - if (check_for_data_type_usage(cluster, typename, output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" - "The \"%s\" type has been removed in PostgreSQL version %s,\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns, or change them to another data type, and restart\n" - "the upgrade. A list of the problem columns is in the file:\n" - " %s", datatype, datatype, version, output_path); - } - else - check_ok(); -} - - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index d9a848cbfd..7daa67f809 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -351,6 +351,9 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + /* * Global variables */ @@ -475,18 +478,10 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 13b2c0f012..e3c7b4109c 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,236 +9,23 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - -/* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. - */ -bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) -{ - bool found = false; - FILE *script = NULL; - int dbnum; - - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %s", output_path, - strerror(errno)); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } - - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; -} - /* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - return found; -} - - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + return false; } /* @@ -353,41 +140,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-03-18 12:11 Daniel Gustafsson <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Daniel Gustafsson @ 2024-03-18 12:11 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: Nathan Bossart <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]> Attached is a fresh rebase with only minor cosmetic touch-ups which I would like to go ahead with during this CF. Peter: does this address the comments you had on translation and code duplication? -- Daniel Gustafsson Attachments: [application/octet-stream] v15-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch (42.0K, ../../[email protected]/2-v15-0001-pg_upgrade-run-all-data-type-checks-per-connecti.patch) download | inline diff: From 428da9b257e73ef01abe3f2ffff218066b0afdaa Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson <[email protected]> Date: Mon, 18 Mar 2024 10:54:45 +0100 Subject: [PATCH v15] pg_upgrade: run all data type checks per connection The checks for data type usage were each connecting to all databases in the cluster and running their query. On clusters which have a lot of databases this can become unnecessarily expensive. This moves the checks to run in a single connection instead to minimize setup and teardown overhead. Reviewed-by: Nathan Bossart <[email protected]> Reviewed-by: Justin Pryzby <[email protected]> Reviewed-by: Peter Eisentraut <[email protected]> Discussion: https://postgr.es/m/[email protected] --- src/bin/pg_upgrade/check.c | 749 +++++++++++++++++++++----------- src/bin/pg_upgrade/pg_upgrade.h | 13 +- src/bin/pg_upgrade/version.c | 265 +---------- 3 files changed, 506 insertions(+), 521 deletions(-) diff --git a/src/bin/pg_upgrade/check.c b/src/bin/pg_upgrade/check.c index 8ce6c674e3..c198896c9f 100644 --- a/src/bin/pg_upgrade/check.c +++ b/src/bin/pg_upgrade/check.c @@ -10,6 +10,7 @@ #include "postgres_fe.h" #include "catalog/pg_authid_d.h" +#include "catalog/pg_class_d.h" #include "catalog/pg_collation.h" #include "fe_utils/string_utils.h" #include "mb/pg_wchar.h" @@ -23,13 +24,6 @@ static void check_for_isn_and_int8_passing_mismatch(ClusterInfo *cluster); static void check_for_user_defined_postfix_ops(ClusterInfo *cluster); static void check_for_incompatible_polymorphics(ClusterInfo *cluster); static void check_for_tables_with_oids(ClusterInfo *cluster); -static void check_for_composite_data_type_usage(ClusterInfo *cluster); -static void check_for_reg_data_type_usage(ClusterInfo *cluster); -static void check_for_aclitem_data_type_usage(ClusterInfo *cluster); -static void check_for_removed_data_type_usage(ClusterInfo *cluster, - const char *version, - const char *datatype); -static void check_for_jsonb_9_4_usage(ClusterInfo *cluster); static void check_for_pg_role_prefix(ClusterInfo *cluster); static void check_for_new_tablespace_dir(void); static void check_for_user_defined_encoding_conversions(ClusterInfo *cluster); @@ -38,6 +32,497 @@ static void check_new_cluster_subscription_configuration(void); static void check_old_cluster_for_valid_slots(bool live_check); static void check_old_cluster_subscription_state(void); +/* + * DataTypesUsageChecks - definitions of data type checks for the old cluster + * in order to determine if an upgrade can be performed. See the comment on + * data_types_usage_checks below for a more detailed description. + */ +typedef struct +{ + /* Status line to print to the user */ + const char *status; + /* Filename to store report to */ + const char *report_filename; + /* Query to extract the oid of the datatype */ + const char *base_query; + /* Text to store to report in case of error */ + const char *report_text; + /* The latest version where the check applies */ + int threshold_version; + /* A function pointer for determining if the check applies */ + DataTypesUsageVersionCheck version_hook; +} DataTypesUsageChecks; + +/* + * Special values for threshold_version for indicating that a check applies to + * all versions, or that a custom function needs to be invoked to determine + * if the check applies. + */ +#define MANUAL_CHECK 1 +#define ALL_VERSIONS -1 + +/*-- + * Data type usage checks. Each check for problematic data type usage is + * defined in this array with metadata, SQL query for finding the data type + * and functionality for deciding if the check is applicable to the version + * of the old cluster. The struct members are described in detail below: + * + * status A oneline string which can be printed to the user to + * inform about progress. Should not end with newline. + * report_filename The filename in which the list of problems detected by + * the check will be printed. + * base_query A query which extracts the Oid of the datatype checked + * for. + * report_text The text which will be printed to the user to explain + * what the check did, and why it failed. The text should + * end with a newline, and does not need to refer to the + * report_filename as that is automatically appended to + * the report with the path to the log folder. + * threshold_version The major version of PostgreSQL for which to run the + * check. Iff the old cluster is less than, or equal to, + * the threshold version then the check will be executed. + * If the old version is greater than the threshold then + * the check is skipped. If the threshold_version is set + * to ALL_VERSIONS then it will be run unconditionally, + * if set to MANUAL_CHECK then the version_hook function + * will be executed in order to determine whether or not + * to run. + * version_hook A function pointer to a version check function of type + * DataTypesUsageVersionCheck which is used to determine + * if the check is applicable to the old cluster. If the + * version_hook returns true then the check will be run, + * else it will be skipped. The function will only be + * executed iff threshold_version is set to MANUAL_CHECK. + */ +static DataTypesUsageChecks data_types_usage_checks[] = +{ + /* + * Look for composite types that were made during initdb *or* belong to + * information_schema; that's important in case information_schema was + * dropped and reloaded. + * + * The cutoff OID here should match the source cluster's value of + * FirstNormalObjectId. We hardcode it rather than using that C #define + * because, if that #define is ever changed, our own version's value is + * NOT what to use. Eventually we may need a test on the source cluster's + * version to select the correct value. + */ + { + .status = gettext_noop("Checking for system-defined composite types in user tables"), + .report_filename = "tables_using_composite.txt", + .base_query = + "SELECT t.oid FROM pg_catalog.pg_type t " + "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " + " WHERE typtype = 'c' AND (t.oid < 16384 OR nspname = 'information_schema')", + .report_text = + gettext_noop("Your installation contains system-defined composite types in user tables.\n" + "These type OIDs are not stable across PostgreSQL versions,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade.\n"), + .threshold_version = ALL_VERSIONS + }, + + /* + * 9.3 -> 9.4 Fully implement the 'line' data type in 9.4, which + * previously returned "not enabled" by default and was only functionally + * enabled with a compile-time switch; as of 9.4 "line" has a different + * on-disk representation format. + */ + { + .status = gettext_noop("Checking for incompatible \"line\" data type"), + .report_filename = "tables_using_line.txt", + .base_query = + "SELECT 'pg_catalog.line'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"line\" data type in user tables.\n" + "this data type changed its internal and input/output format\n" + "between your old and new versions so this\n" + "cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n"), + .threshold_version = 903 + }, + + /* + * pg_upgrade only preserves these system values: pg_class.oid pg_type.oid + * pg_enum.oid + * + * Many of the reg* data types reference system catalog info that is not + * preserved, and hence these data types cannot be used in user tables + * upgraded by pg_upgrade. + */ + { + .status = gettext_noop("Checking for reg* data types in user tables"), + .report_filename = "tables_using_reg.txt", + + /* + * Note: older servers will not have all of these reg* types, so we + * have to write the query like this rather than depending on casts to + * regtype. + */ + .base_query = + "SELECT oid FROM pg_catalog.pg_type t " + "WHERE t.typnamespace = " + " (SELECT oid FROM pg_catalog.pg_namespace " + " WHERE nspname = 'pg_catalog') " + " AND t.typname IN ( " + /* pg_class.oid is preserved, so 'regclass' is OK */ + " 'regcollation', " + " 'regconfig', " + " 'regdictionary', " + " 'regnamespace', " + " 'regoper', " + " 'regoperator', " + " 'regproc', " + " 'regprocedure' " + /* pg_authid.oid is preserved, so 'regrole' is OK */ + /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ + " )", + .report_text = + gettext_noop("Your installation contains one of the reg* data types in user tables.\n" + "These data types reference system OIDs that are not preserved by\n" + "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" + "drop the problem columns and restart the upgrade.\n"), + .threshold_version = ALL_VERSIONS + }, + + /* + * PG 16 increased the size of the 'aclitem' type, which breaks the + * on-disk format for existing data. + */ + { + .status = gettext_noop("Checking for incompatible \"aclitem\" data type"), + .report_filename = "tables_using_aclitem.txt", + .base_query = + "SELECT 'pg_catalog.aclitem'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"aclitem\" data type in user tables.\n" + "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns and restart the upgrade.\n"), + .threshold_version = 1500 + }, + + /* + * It's no longer allowed to create tables or views with "unknown"-type + * columns. We do not complain about views with such columns, because + * they should get silently converted to "text" columns during the DDL + * dump and reload; it seems unlikely to be worth making users do that by + * hand. However, if there's a table with such a column, the DDL reload + * will fail, so we should pre-detect that rather than failing + * mid-upgrade. Worse, if there's a matview with such a column, the DDL + * reload will silently change it to "text" which won't match the on-disk + * storage (which is like "cstring"). So we *must* reject that. + */ + { + .status = gettext_noop("Checking for invalid \"unknown\" user columns"), + .report_filename = "tables_using_unknown.txt", + .base_query = + "SELECT 'pg_catalog.unknown'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"unknown\" data type in user tables.\n" + "This data type is no longer allowed in tables, so this cluster\n" + "cannot currently be upgraded. You can drop the problem columns\n" + "and restart the upgrade.\n"), + .threshold_version = 906 + }, + + /* + * PG 12 changed the 'sql_identifier' type storage to be based on name, + * not varchar, which breaks on-disk format for existing data. So we need + * to prevent upgrade when used in user objects (tables, indexes, ...). In + * 12, the sql_identifier data type was switched from name to varchar, + * which does affect the storage (name is by-ref, but not varlena). This + * means user tables using sql_identifier for columns are broken because + * the on-disk format is different. + */ + { + .status = gettext_noop("Checking for invalid \"sql_identifier\" user columns"), + .report_filename = "tables_using_sql_identifier.txt", + .base_query = + "SELECT 'information_schema.sql_identifier'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"sql_identifier\" data type in user tables.\n" + "The on-disk format for this data type has changed, so this\n" + "cluster cannot currently be upgraded. You can drop the problem\n" + "columns and restart the upgrade.\n"), + .threshold_version = 1100 + }, + + /* + * JSONB changed its storage format during 9.4 beta, so check for it. + */ + { + .status = gettext_noop("Checking for incompatible \"jsonb\" data type in user tables"), + .report_filename = "tables_using_jsonb.txt", + .base_query = + "SELECT 'pg_catalog.jsonb'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"jsonb\" data type in user tables.\n" + "The internal format of \"jsonb\" changed during 9.4 beta so this\n" + "cluster cannot currently be upgraded. You can drop the problem \n" + "columns and restart the upgrade.\n"), + .threshold_version = MANUAL_CHECK, + .version_hook = jsonb_9_4_check_applicable + }, + + /* + * PG 12 removed types abstime, reltime, tinterval. + */ + { + .status = gettext_noop("Checking for removed \"abstime\" data type in user tables"), + .report_filename = "tables_using_abstime.txt", + .base_query = + "SELECT 'pg_catalog.abstime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"abstime\" data type in user tables.\n" + "The \"abstime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade.\n"), + .threshold_version = 1100 + }, + { + .status = gettext_noop("Checking for removed \"reltime\" data type in user tables"), + .report_filename = "tables_using_reltime.txt", + .base_query = + "SELECT 'pg_catalog.reltime'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"reltime\" data type in user tables.\n" + "The \"reltime\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade.\n"), + .threshold_version = 1100 + }, + { + .status = gettext_noop("Checking for removed \"tinterval\" data type in user tables"), + .report_filename = "tables_using_tinterval.txt", + .base_query = + "SELECT 'pg_catalog.tinterval'::pg_catalog.regtype AS oid", + .report_text = + gettext_noop("Your installation contains the \"tinterval\" data type in user tables.\n" + "The \"tinterval\" type has been removed in PostgreSQL version 12,\n" + "so this cluster cannot currently be upgraded. You can drop the\n" + "problem columns, or change them to another data type, and restart\n" + "the upgrade.\n"), + .threshold_version = 1100 + }, + + /* End of checks marker, must remain last */ + { + NULL, NULL, NULL, NULL, 0, NULL + } +}; + +/* + * check_for_data_types_usage() + * Detect whether there are any stored columns depending on given type(s) + * + * If so, write a report to the given file name and signal a failure to the + * user. + * + * The checks to run are defined in a DataTypesUsageChecks structure where + * each check has a metadata for explaining errors to the user, a base_query, + * a report filename and a function pointer hook for validating if the check + * should be executed given the cluster at hand. + * + * base_query should be a SELECT yielding a single column named "oid", + * containing the pg_type OIDs of one or more types that are known to have + * inconsistent on-disk representations across server versions. + * + * We check for the type(s) in tables, matviews, and indexes, but not views; + * there's no storage involved in a view. + */ +static void +check_for_data_types_usage(ClusterInfo *cluster, DataTypesUsageChecks * checks) +{ + bool found = false; + bool *results; + PQExpBufferData report; + DataTypesUsageChecks *tmp = checks; + int n_data_types_usage_checks = 0; + + prep_status("Checking for data type usage"); + + /* Gather number of checks to perform */ + while (tmp->status != NULL) + { + n_data_types_usage_checks++; + tmp++; + } + + /* Prepare an array to store the results of checks in */ + results = pg_malloc0(sizeof(bool) * n_data_types_usage_checks); + + /* + * Connect to each database in the cluster and run all defined checks + * against that database before trying the next one. + */ + for (int dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) + { + DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; + PGconn *conn = connectToServer(cluster, active_db->db_name); + + for (int checknum = 0; checknum < n_data_types_usage_checks; checknum++) + { + PGresult *res; + int ntups; + int i_nspname; + int i_relname; + int i_attname; + FILE *script = NULL; + bool db_used = false; + char output_path[MAXPGPATH]; + DataTypesUsageChecks *cur_check = &checks[checknum]; + + if (cur_check->threshold_version == MANUAL_CHECK) + { + Assert(cur_check->version_hook); + + /* + * Make sure that the check applies to the current cluster + * version and skip if not. If no check hook has been defined + * we run the check for all versions. + */ + if (!cur_check->version_hook(cluster)) + continue; + } + else if (cur_check->threshold_version != ALL_VERSIONS) + { + if (GET_MAJOR_VERSION(cluster->major_version) > cur_check->threshold_version) + continue; + } + else + Assert(cur_check->threshold_version == ALL_VERSIONS); + + snprintf(output_path, sizeof(output_path), "%s/%s", + log_opts.basedir, + cur_check->report_filename); + + /* + * The type(s) of interest might be wrapped in a domain, array, + * composite, or range, and these container types can be nested + * (to varying extents depending on server version, but that's not + * of concern here). To handle all these cases we need a + * recursive CTE. + */ + res = executeQueryOrDie(conn, + "WITH RECURSIVE oids AS ( " + /* start with the type(s) returned by base_query */ + " %s " + " UNION ALL " + " SELECT * FROM ( " + /* inner WITH because we can only reference the CTE once */ + " WITH x AS (SELECT oid FROM oids) " + /* domains on any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " + " UNION ALL " + /* arrays over any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " + " UNION ALL " + /* composite types containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " + " WHERE t.typtype = 'c' AND " + " t.oid = c.reltype AND " + " c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid = x.oid " + " UNION ALL " + /* ranges containing any type selected so far */ + " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " + " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" + " ) foo " + ") " + /* now look for stored columns of any such type */ + "SELECT n.nspname, c.relname, a.attname " + "FROM pg_catalog.pg_class c, " + " pg_catalog.pg_namespace n, " + " pg_catalog.pg_attribute a " + "WHERE c.oid = a.attrelid AND " + " NOT a.attisdropped AND " + " a.atttypid IN (SELECT oid FROM oids) AND " + " c.relkind IN (" + CppAsString2(RELKIND_RELATION) ", " + CppAsString2(RELKIND_MATVIEW) ", " + CppAsString2(RELKIND_INDEX) ") AND " + " c.relnamespace = n.oid AND " + /* exclude possible orphaned temp tables */ + " n.nspname !~ '^pg_temp_' AND " + " n.nspname !~ '^pg_toast_temp_' AND " + /* exclude system catalogs, too */ + " n.nspname NOT IN ('pg_catalog', 'information_schema')", + cur_check->base_query); + + ntups = PQntuples(res); + + /* + * The datatype was found, so extract the data and log to the + * requested filename. We need to open the file for appending + * since the check might have already found the type in another + * database earlier in the loop. + */ + if (ntups) + { + /* + * Make sure we have a buffer to save reports to now that we + * found a first failing check. + */ + if (!found) + initPQExpBuffer(&report); + found = true; + + /* + * If this is the first time we see an error for the check in + * question then print a status message of the failure. + */ + if (!results[checknum]) + { + pg_log(PG_REPORT, " failed check: %s", _(cur_check->status)); + appendPQExpBuffer(&report, "\n%s\n%s %s\n", + _(cur_check->report_text), + _("A list of the problem columns is in the file:"), + output_path); + } + results[checknum] = true; + + i_nspname = PQfnumber(res, "nspname"); + i_relname = PQfnumber(res, "relname"); + i_attname = PQfnumber(res, "attname"); + + for (int rowno = 0; rowno < ntups; rowno++) + { + if (script == NULL && (script = fopen_priv(output_path, "a")) == NULL) + pg_fatal("could not open file \"%s\": %m", output_path); + + if (!db_used) + { + fprintf(script, "In database: %s\n", active_db->db_name); + db_used = true; + } + fprintf(script, " %s.%s.%s\n", + PQgetvalue(res, rowno, i_nspname), + PQgetvalue(res, rowno, i_relname), + PQgetvalue(res, rowno, i_attname)); + } + + if (script) + { + fclose(script); + script = NULL; + } + } + + PQclear(res); + } + + PQfinish(conn); + } + + if (found) + pg_fatal("Data type checks failed: %s", report.data); + + check_ok(); +} /* * fix_path_separator @@ -110,8 +595,6 @@ check_and_dump_old_cluster(bool live_check) check_is_install_user(&old_cluster); check_proper_datallowconn(&old_cluster); check_for_prepared_transactions(&old_cluster); - check_for_composite_data_type_usage(&old_cluster); - check_for_reg_data_type_usage(&old_cluster); check_for_isn_and_int8_passing_mismatch(&old_cluster); if (GET_MAJOR_VERSION(old_cluster.major_version) >= 1700) @@ -129,22 +612,7 @@ check_and_dump_old_cluster(bool live_check) check_old_cluster_subscription_state(); } - /* - * PG 16 increased the size of the 'aclitem' type, which breaks the - * on-disk format for existing data. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1500) - check_for_aclitem_data_type_usage(&old_cluster); - - /* - * PG 12 removed types abstime, reltime, tinterval. - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - { - check_for_removed_data_type_usage(&old_cluster, "12", "abstime"); - check_for_removed_data_type_usage(&old_cluster, "12", "reltime"); - check_for_removed_data_type_usage(&old_cluster, "12", "tinterval"); - } + check_for_data_types_usage(&old_cluster, data_types_usage_checks); /* * PG 14 changed the function signature of encoding conversion functions. @@ -176,21 +644,12 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) check_for_tables_with_oids(&old_cluster); - /* - * PG 12 changed the 'sql_identifier' type storage to be based on name, - * not varchar, which breaks on-disk format for existing data. So we need - * to prevent upgrade when used in user objects (tables, indexes, ...). - */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 1100) - old_11_check_for_sql_identifier_data_type_usage(&old_cluster); - /* * Pre-PG 10 allowed tables with 'unknown' type columns and non WAL logged * hash indexes */ if (GET_MAJOR_VERSION(old_cluster.major_version) <= 906) { - old_9_6_check_for_unknown_data_type_usage(&old_cluster); if (user_opts.check) old_9_6_invalidate_hash_indexes(&old_cluster, true); } @@ -199,14 +658,6 @@ check_and_dump_old_cluster(bool live_check) if (GET_MAJOR_VERSION(old_cluster.major_version) <= 905) check_for_pg_role_prefix(&old_cluster); - if (GET_MAJOR_VERSION(old_cluster.major_version) == 904 && - old_cluster.controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) - check_for_jsonb_9_4_usage(&old_cluster); - - /* Pre-PG 9.4 had a different 'line' data type internal format */ - if (GET_MAJOR_VERSION(old_cluster.major_version) <= 903) - old_9_3_check_for_line_data_type_usage(&old_cluster); - /* * While not a check option, we do this now because this is the only time * the old server is running. @@ -1122,220 +1573,6 @@ check_for_tables_with_oids(ClusterInfo *cluster) } -/* - * check_for_composite_data_type_usage() - * Check for system-defined composite types used in user tables. - * - * The OIDs of rowtypes of system catalogs and information_schema views - * can change across major versions; unlike user-defined types, we have - * no mechanism for forcing them to be the same in the new cluster. - * Hence, if any user table uses one, that's problematic for pg_upgrade. - */ -static void -check_for_composite_data_type_usage(ClusterInfo *cluster) -{ - bool found; - Oid firstUserOid; - char output_path[MAXPGPATH]; - char *base_query; - - prep_status("Checking for system-defined composite types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_composite.txt"); - - /* - * Look for composite types that were made during initdb *or* belong to - * information_schema; that's important in case information_schema was - * dropped and reloaded. - * - * The cutoff OID here should match the source cluster's value of - * FirstNormalObjectId. We hardcode it rather than using that C #define - * because, if that #define is ever changed, our own version's value is - * NOT what to use. Eventually we may need a test on the source cluster's - * version to select the correct value. - */ - firstUserOid = 16384; - - base_query = psprintf("SELECT t.oid FROM pg_catalog.pg_type t " - "LEFT JOIN pg_catalog.pg_namespace n ON t.typnamespace = n.oid " - " WHERE typtype = 'c' AND (t.oid < %u OR nspname = 'information_schema')", - firstUserOid); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains system-defined composite types in user tables.\n" - "These type OIDs are not stable across PostgreSQL versions,\n" - "so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_reg_data_type_usage() - * pg_upgrade only preserves these system values: - * pg_class.oid - * pg_type.oid - * pg_enum.oid - * - * Many of the reg* data types reference system catalog info that is - * not preserved, and hence these data types cannot be used in user - * tables upgraded by pg_upgrade. - */ -static void -check_for_reg_data_type_usage(ClusterInfo *cluster) -{ - bool found; - char output_path[MAXPGPATH]; - - prep_status("Checking for reg* data types in user tables"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_reg.txt"); - - /* - * Note: older servers will not have all of these reg* types, so we have - * to write the query like this rather than depending on casts to regtype. - */ - found = check_for_data_types_usage(cluster, - "SELECT oid FROM pg_catalog.pg_type t " - "WHERE t.typnamespace = " - " (SELECT oid FROM pg_catalog.pg_namespace " - " WHERE nspname = 'pg_catalog') " - " AND t.typname IN ( " - /* pg_class.oid is preserved, so 'regclass' is OK */ - " 'regcollation', " - " 'regconfig', " - " 'regdictionary', " - " 'regnamespace', " - " 'regoper', " - " 'regoperator', " - " 'regproc', " - " 'regprocedure' " - /* pg_authid.oid is preserved, so 'regrole' is OK */ - /* pg_type.oid is (mostly) preserved, so 'regtype' is OK */ - " )", - output_path); - - if (found) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains one of the reg* data types in user tables.\n" - "These data types reference system OIDs that are not preserved by\n" - "pg_upgrade, so this cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_aclitem_data_type_usage - * - * aclitem changed its storage format in 16, so check for it. - */ -static void -check_for_aclitem_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"%s\" data type in user tables", - "aclitem"); - - snprintf(output_path, sizeof(output_path), "tables_using_aclitem.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.aclitem", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"aclitem\" data type in user tables.\n" - "The internal format of \"aclitem\" changed in PostgreSQL version 16\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns and restart the upgrade. A list of the problem\n" - "columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - -/* - * check_for_removed_data_type_usage - * - * Check for in-core data types that have been removed. Callers know - * the exact list. - */ -static void -check_for_removed_data_type_usage(ClusterInfo *cluster, const char *version, - const char *datatype) -{ - char output_path[MAXPGPATH]; - char typename[NAMEDATALEN]; - - prep_status("Checking for removed \"%s\" data type in user tables", - datatype); - - snprintf(output_path, sizeof(output_path), "tables_using_%s.txt", - datatype); - snprintf(typename, sizeof(typename), "pg_catalog.%s", datatype); - - if (check_for_data_type_usage(cluster, typename, output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"%s\" data type in user tables.\n" - "The \"%s\" type has been removed in PostgreSQL version %s,\n" - "so this cluster cannot currently be upgraded. You can drop the\n" - "problem columns, or change them to another data type, and restart\n" - "the upgrade. A list of the problem columns is in the file:\n" - " %s", datatype, datatype, version, output_path); - } - else - check_ok(); -} - - -/* - * check_for_jsonb_9_4_usage() - * - * JSONB changed its storage format during 9.4 beta, so check for it. - */ -static void -check_for_jsonb_9_4_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"jsonb\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_jsonb.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.jsonb", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"jsonb\" data type in user tables.\n" - "The internal format of \"jsonb\" changed during 9.4 beta so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - /* * check_for_pg_role_prefix() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index c0bfb002d2..92bcb693fb 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -352,6 +352,9 @@ typedef struct } OSInfo; +/* Function signature for data type check version hook */ +typedef bool (*DataTypesUsageVersionCheck) (ClusterInfo *cluster); + /* * Global variables */ @@ -479,18 +482,10 @@ unsigned int str2uint(const char *str); /* version.c */ -bool check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path); -bool check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path); -void old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster); -void old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster); +bool jsonb_9_4_check_applicable(ClusterInfo *cluster); void old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode); -void old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster); void report_extension_updates(ClusterInfo *cluster); /* parallel.c */ diff --git a/src/bin/pg_upgrade/version.c b/src/bin/pg_upgrade/version.c index 9dc1399f36..2de6dffccd 100644 --- a/src/bin/pg_upgrade/version.c +++ b/src/bin/pg_upgrade/version.c @@ -9,235 +9,23 @@ #include "postgres_fe.h" -#include "catalog/pg_class_d.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" - /* - * check_for_data_types_usage() - * Detect whether there are any stored columns depending on given type(s) - * - * If so, write a report to the given file name, and return true. - * - * base_query should be a SELECT yielding a single column named "oid", - * containing the pg_type OIDs of one or more types that are known to have - * inconsistent on-disk representations across server versions. - * - * We check for the type(s) in tables, matviews, and indexes, but not views; - * there's no storage involved in a view. + * version_hook functions for check_for_data_types_usage in order to determine + * whether a data type check should be executed for the cluster in question or + * not. */ bool -check_for_data_types_usage(ClusterInfo *cluster, - const char *base_query, - const char *output_path) +jsonb_9_4_check_applicable(ClusterInfo *cluster) { - bool found = false; - FILE *script = NULL; - int dbnum; + /* JSONB changed its storage format during 9.4 beta */ + if (GET_MAJOR_VERSION(cluster->major_version) == 904 && + cluster->controldata.cat_ver < JSONB_FORMAT_CHANGE_CAT_VER) + return true; - for (dbnum = 0; dbnum < cluster->dbarr.ndbs; dbnum++) - { - DbInfo *active_db = &cluster->dbarr.dbs[dbnum]; - PGconn *conn = connectToServer(cluster, active_db->db_name); - PQExpBufferData querybuf; - PGresult *res; - bool db_used = false; - int ntups; - int rowno; - int i_nspname, - i_relname, - i_attname; - - /* - * The type(s) of interest might be wrapped in a domain, array, - * composite, or range, and these container types can be nested (to - * varying extents depending on server version, but that's not of - * concern here). To handle all these cases we need a recursive CTE. - */ - initPQExpBuffer(&querybuf); - appendPQExpBuffer(&querybuf, - "WITH RECURSIVE oids AS ( " - /* start with the type(s) returned by base_query */ - " %s " - " UNION ALL " - " SELECT * FROM ( " - /* inner WITH because we can only reference the CTE once */ - " WITH x AS (SELECT oid FROM oids) " - /* domains on any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typbasetype = x.oid AND typtype = 'd' " - " UNION ALL " - /* arrays over any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, x WHERE typelem = x.oid AND typtype = 'b' " - " UNION ALL " - /* composite types containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_class c, pg_catalog.pg_attribute a, x " - " WHERE t.typtype = 'c' AND " - " t.oid = c.reltype AND " - " c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid = x.oid " - " UNION ALL " - /* ranges containing any type selected so far */ - " SELECT t.oid FROM pg_catalog.pg_type t, pg_catalog.pg_range r, x " - " WHERE t.typtype = 'r' AND r.rngtypid = t.oid AND r.rngsubtype = x.oid" - " ) foo " - ") " - /* now look for stored columns of any such type */ - "SELECT n.nspname, c.relname, a.attname " - "FROM pg_catalog.pg_class c, " - " pg_catalog.pg_namespace n, " - " pg_catalog.pg_attribute a " - "WHERE c.oid = a.attrelid AND " - " NOT a.attisdropped AND " - " a.atttypid IN (SELECT oid FROM oids) AND " - " c.relkind IN (" - CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_MATVIEW) ", " - CppAsString2(RELKIND_INDEX) ") AND " - " c.relnamespace = n.oid AND " - /* exclude possible orphaned temp tables */ - " n.nspname !~ '^pg_temp_' AND " - " n.nspname !~ '^pg_toast_temp_' AND " - /* exclude system catalogs, too */ - " n.nspname NOT IN ('pg_catalog', 'information_schema')", - base_query); - - res = executeQueryOrDie(conn, "%s", querybuf.data); - - ntups = PQntuples(res); - i_nspname = PQfnumber(res, "nspname"); - i_relname = PQfnumber(res, "relname"); - i_attname = PQfnumber(res, "attname"); - for (rowno = 0; rowno < ntups; rowno++) - { - found = true; - if (script == NULL && (script = fopen_priv(output_path, "w")) == NULL) - pg_fatal("could not open file \"%s\": %m", output_path); - if (!db_used) - { - fprintf(script, "In database: %s\n", active_db->db_name); - db_used = true; - } - fprintf(script, " %s.%s.%s\n", - PQgetvalue(res, rowno, i_nspname), - PQgetvalue(res, rowno, i_relname), - PQgetvalue(res, rowno, i_attname)); - } - - PQclear(res); - - termPQExpBuffer(&querybuf); - - PQfinish(conn); - } - - if (script) - fclose(script); - - return found; -} - -/* - * check_for_data_type_usage() - * Detect whether there are any stored columns depending on the given type - * - * If so, write a report to the given file name, and return true. - * - * type_name should be a fully qualified type name. This is just a - * trivial wrapper around check_for_data_types_usage() to convert a - * type name into a base query. - */ -bool -check_for_data_type_usage(ClusterInfo *cluster, - const char *type_name, - const char *output_path) -{ - bool found; - char *base_query; - - base_query = psprintf("SELECT '%s'::pg_catalog.regtype AS oid", - type_name); - - found = check_for_data_types_usage(cluster, base_query, output_path); - - free(base_query); - - return found; -} - - -/* - * old_9_3_check_for_line_data_type_usage() - * 9.3 -> 9.4 - * Fully implement the 'line' data type in 9.4, which previously returned - * "not enabled" by default and was only functionally enabled with a - * compile-time switch; as of 9.4 "line" has a different on-disk - * representation format. - */ -void -old_9_3_check_for_line_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for incompatible \"line\" data type"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_line.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.line", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"line\" data type in user tables.\n" - "This data type changed its internal and input/output format\n" - "between your old and new versions so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - -/* - * old_9_6_check_for_unknown_data_type_usage() - * 9.6 -> 10 - * It's no longer allowed to create tables or views with "unknown"-type - * columns. We do not complain about views with such columns, because - * they should get silently converted to "text" columns during the DDL - * dump and reload; it seems unlikely to be worth making users do that - * by hand. However, if there's a table with such a column, the DDL - * reload will fail, so we should pre-detect that rather than failing - * mid-upgrade. Worse, if there's a matview with such a column, the - * DDL reload will silently change it to "text" which won't match the - * on-disk storage (which is like "cstring"). So we *must* reject that. - */ -void -old_9_6_check_for_unknown_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"unknown\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_unknown.txt"); - - if (check_for_data_type_usage(cluster, "pg_catalog.unknown", output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"unknown\" data type in user tables.\n" - "This data type is no longer allowed in tables, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); + return false; } /* @@ -351,41 +139,6 @@ old_9_6_invalidate_hash_indexes(ClusterInfo *cluster, bool check_mode) check_ok(); } -/* - * old_11_check_for_sql_identifier_data_type_usage() - * 11 -> 12 - * In 12, the sql_identifier data type was switched from name to varchar, - * which does affect the storage (name is by-ref, but not varlena). This - * means user tables using sql_identifier for columns are broken because - * the on-disk format is different. - */ -void -old_11_check_for_sql_identifier_data_type_usage(ClusterInfo *cluster) -{ - char output_path[MAXPGPATH]; - - prep_status("Checking for invalid \"sql_identifier\" user columns"); - - snprintf(output_path, sizeof(output_path), "%s/%s", - log_opts.basedir, - "tables_using_sql_identifier.txt"); - - if (check_for_data_type_usage(cluster, "information_schema.sql_identifier", - output_path)) - { - pg_log(PG_REPORT, "fatal"); - pg_fatal("Your installation contains the \"sql_identifier\" data type in user tables.\n" - "The on-disk format for this data type has changed, so this\n" - "cluster cannot currently be upgraded. You can\n" - "drop the problem columns and restart the upgrade.\n" - "A list of the problem columns is in the file:\n" - " %s", output_path); - } - else - check_ok(); -} - - /* * report_extension_updates() * Report extensions that should be updated. -- 2.32.1 (Apple Git-133) ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-03-19 07:07 Peter Eisentraut <[email protected]> parent: Daniel Gustafsson <[email protected]> 0 siblings, 1 reply; 25+ messages in thread From: Peter Eisentraut @ 2024-03-19 07:07 UTC (permalink / raw) To: Daniel Gustafsson <[email protected]>; +Cc: Nathan Bossart <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]> On 18.03.24 13:11, Daniel Gustafsson wrote: > Attached is a fresh rebase with only minor cosmetic touch-ups which I would > like to go ahead with during this CF. > > Peter: does this address the comments you had on translation and code > duplication? Yes, this looks good. ^ permalink raw reply [nested|flat] 25+ messages in thread
* Re: Reducing connection overhead in pg_upgrade compat check phase @ 2024-03-19 13:38 Daniel Gustafsson <[email protected]> parent: Peter Eisentraut <[email protected]> 0 siblings, 0 replies; 25+ messages in thread From: Daniel Gustafsson @ 2024-03-19 13:38 UTC (permalink / raw) To: Peter Eisentraut <[email protected]>; +Cc: Nathan Bossart <[email protected]>; vignesh C <[email protected]>; PostgreSQL Hackers <[email protected]> > On 19 Mar 2024, at 08:07, Peter Eisentraut <[email protected]> wrote: > > On 18.03.24 13:11, Daniel Gustafsson wrote: >> Attached is a fresh rebase with only minor cosmetic touch-ups which I would >> like to go ahead with during this CF. >> Peter: does this address the comments you had on translation and code >> duplication? > > Yes, this looks good. Thanks for review! I took another look at this and pushed it. -- Daniel Gustafsson ^ permalink raw reply [nested|flat] 25+ messages in thread
end of thread, other threads:[~2024-03-19 13:38 UTC | newest] Thread overview: 25+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-12-20 01:21 [PATCH v30 05/11] Add Incremental View Maintenance support to psql Yugo Nagata <[email protected]> 2023-07-06 15:58 Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2023-07-08 21:43 ` Re: Reducing connection overhead in pg_upgrade compat check phase Nathan Bossart <[email protected]> 2023-07-10 14:43 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2023-07-10 23:09 ` Re: Reducing connection overhead in pg_upgrade compat check phase Nathan Bossart <[email protected]> 2023-07-10 23:26 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2023-07-11 22:43 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2023-07-11 23:36 ` Re: Reducing connection overhead in pg_upgrade compat check phase Nathan Bossart <[email protected]> 2023-08-31 21:34 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2023-09-13 14:12 ` Re: Reducing connection overhead in pg_upgrade compat check phase Peter Eisentraut <[email protected]> 2023-09-14 08:48 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2023-10-27 13:20 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2024-01-27 03:40 ` Re: Reducing connection overhead in pg_upgrade compat check phase vignesh C <[email protected]> 2024-02-01 18:48 ` Re: Reducing connection overhead in pg_upgrade compat check phase vignesh C <[email protected]> 2024-02-06 16:32 ` Re: Reducing connection overhead in pg_upgrade compat check phase Nathan Bossart <[email protected]> 2024-02-06 16:47 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2024-02-06 16:55 ` Re: Reducing connection overhead in pg_upgrade compat check phase Nathan Bossart <[email protected]> 2024-02-07 13:25 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2024-02-08 10:55 ` Re: Reducing connection overhead in pg_upgrade compat check phase Peter Eisentraut <[email protected]> 2024-02-08 14:16 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2024-02-08 23:04 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2024-02-09 09:33 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2024-03-18 12:11 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]> 2024-03-19 07:07 ` Re: Reducing connection overhead in pg_upgrade compat check phase Peter Eisentraut <[email protected]> 2024-03-19 13:38 ` Re: Reducing connection overhead in pg_upgrade compat check phase Daniel Gustafsson <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox