public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 1/2] bootstrap: convert Typ to a List* 8+ messages / 4 participants [nested] [flat]
* [PATCH 1/2] bootstrap: convert Typ to a List* @ 2020-11-20 02:48 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 8+ messages in thread From: Justin Pryzby @ 2020-11-20 02:48 UTC (permalink / raw) --- src/backend/bootstrap/bootstrap.c | 89 ++++++++++++++----------------- 1 file changed, 41 insertions(+), 48 deletions(-) diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 6f615e6622..1b940d9d27 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -58,7 +58,7 @@ static void BootstrapModeMain(void); static void bootstrap_signals(void); static void ShutdownAuxiliaryProcess(int code, Datum arg); static Form_pg_attribute AllocateAttribute(void); -static void populate_typ_array(void); +static void populate_typ(void); static Oid gettype(char *type); static void cleanup(void); @@ -159,7 +159,7 @@ struct typmap FormData_pg_type am_typ; }; -static struct typmap **Typ = NULL; +static List *Typ = NIL; /* List of struct typmap* */ static struct typmap *Ap = NULL; static Datum values[MAXATTR]; /* current row's attribute values */ @@ -595,10 +595,10 @@ boot_openrel(char *relname) /* * pg_type must be filled before any OPEN command is executed, hence we - * can now populate the Typ array if we haven't yet. + * can now populate Typ if we haven't yet. */ - if (Typ == NULL) - populate_typ_array(); + if (Typ == NIL) + populate_typ(); if (boot_reldesc != NULL) closerel(NULL); @@ -688,7 +688,7 @@ DefineAttr(char *name, char *type, int attnum, int nullness) typeoid = gettype(type); - if (Typ != NULL) + if (Typ != NIL) { attrtypes[attnum]->atttypid = Ap->am_oid; attrtypes[attnum]->attlen = Ap->am_typ.typlen; @@ -866,47 +866,36 @@ cleanup(void) } /* ---------------- - * populate_typ_array + * populate_typ * - * Load the Typ array by reading pg_type. + * Load Typ by reading pg_type. * ---------------- */ static void -populate_typ_array(void) +populate_typ(void) { Relation rel; TableScanDesc scan; HeapTuple tup; - int nalloc; - int i; - - Assert(Typ == NULL); + MemoryContext old; - nalloc = 512; - Typ = (struct typmap **) - MemoryContextAlloc(TopMemoryContext, nalloc * sizeof(struct typmap *)); + Assert(Typ == NIL); rel = table_open(TypeRelationId, NoLock); scan = table_beginscan_catalog(rel, 0, NULL); - i = 0; + old = MemoryContextSwitchTo(TopMemoryContext); while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL) { Form_pg_type typForm = (Form_pg_type) GETSTRUCT(tup); + struct typmap *newtyp; - /* make sure there will be room for a trailing NULL pointer */ - if (i >= nalloc - 1) - { - nalloc *= 2; - Typ = (struct typmap **) - repalloc(Typ, nalloc * sizeof(struct typmap *)); - } - Typ[i] = (struct typmap *) - MemoryContextAlloc(TopMemoryContext, sizeof(struct typmap)); - Typ[i]->am_oid = typForm->oid; - memcpy(&(Typ[i]->am_typ), typForm, sizeof(Typ[i]->am_typ)); - i++; + newtyp = (struct typmap *) palloc(sizeof(struct typmap)); + Typ = lappend(Typ, newtyp); + + newtyp->am_oid = typForm->oid; + memcpy(&newtyp->am_typ, typForm, sizeof(newtyp->am_typ)); } - Typ[i] = NULL; /* Fill trailing NULL pointer */ + MemoryContextSwitchTo(old); table_endscan(scan); table_close(rel, NoLock); } @@ -916,25 +905,26 @@ populate_typ_array(void) * * NB: this is really ugly; it will return an integer index into TypInfo[], * and not an OID at all, until the first reference to a type not known in - * TypInfo[]. At that point it will read and cache pg_type in the Typ array, + * TypInfo[]. At that point it will read and cache pg_type in Typ, * and subsequently return a real OID (and set the global pointer Ap to * point at the found row in Typ). So caller must check whether Typ is - * still NULL to determine what the return value is! + * still NIL to determine what the return value is! * ---------------- */ static Oid gettype(char *type) { - if (Typ != NULL) + if (Typ != NIL) { - struct typmap **app; + ListCell *lc; - for (app = Typ; *app != NULL; app++) + foreach (lc, Typ) { - if (strncmp(NameStr((*app)->am_typ.typname), type, NAMEDATALEN) == 0) + struct typmap *app = lfirst(lc); + if (strncmp(NameStr(app->am_typ.typname), type, NAMEDATALEN) == 0) { - Ap = *app; - return (*app)->am_oid; + Ap = app; + return app->am_oid; } } } @@ -949,7 +939,7 @@ gettype(char *type) } /* Not in TypInfo, so we'd better be able to read pg_type now */ elog(DEBUG4, "external type: %s", type); - populate_typ_array(); + populate_typ(); return gettype(type); } elog(ERROR, "unrecognized type \"%s\"", type); @@ -977,17 +967,20 @@ boot_get_type_io_data(Oid typid, Oid *typinput, Oid *typoutput) { - if (Typ != NULL) + if (Typ != NIL) { /* We have the boot-time contents of pg_type, so use it */ - struct typmap **app; - struct typmap *ap; - - app = Typ; - while (*app && (*app)->am_oid != typid) - ++app; - ap = *app; - if (ap == NULL) + struct typmap *ap = NULL; + ListCell *lc; + + foreach (lc, Typ) + { + ap = lfirst(lc); + if (ap->am_oid == typid) + break; + } + + if (!ap || ap->am_oid != typid) elog(ERROR, "type OID %u not found in Typ list", typid); *typlen = ap->am_typ.typlen; -- 2.17.0 --yQbNiKLmgenwUfTN Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0002-Allow-composite-types-in-bootstrap.patchx" ^ permalink raw reply [nested|flat] 8+ messages in thread
* Re: RFC: Extension Packaging & Lookup @ 2024-10-28 22:19 David E. Wheeler <[email protected]> 0 siblings, 2 replies; 8+ messages in thread From: David E. Wheeler @ 2024-10-28 22:19 UTC (permalink / raw) To: pgsql-hackers; +Cc: Gabriele Bartolini <[email protected]>; Peter Eisentraut <[email protected]>; Christoph Berg <[email protected]>; Andres Freund <[email protected]> Greetings Postgres humans, There was much discussion of this proposal at PGConf.eu <http://pgconf.eu/; last week, between Gabriele Bartolini, Peter Eisentraut, Christoph Berg, and Andres Freund (all Cc’d here), and me, among others. We agreed, in principle, to an approach to this feature. Overall I think the proposal doesn’t need to change, but there are a couple of things to tweak, and I’ve added a list of use cases I’m aware of below, plus a tangent on the challenges of loading system DOS. Quoting a lot and responding inline. On Oct 10, 2024, at 4:34 PM, David E. Wheeler <[email protected]> wrote: > I guess I should get off my butt and do it. So let’s do this. Here’s what I propose. > > * When an extension is installed, all of its files should live in a single directory. These include: > > * The Control file in directory describes extension > * Subdirectories for SQL, shared libraries, docs, binaries > (also locales and tsearch dictionaries?) Just to be clear, these directories correspond to the `pg_config `--*dir` options, excluding include directories: ``` ❯ pg_config --help | grep 'dir\b' | grep -v include --bindir show location of user executables --docdir show location of documentation files --htmldir show location of HTML documentation files --libdir show location of object code libraries --pkglibdir show location of dynamically loadable modules --localedir show location of locale support files --mandir show location of manual pages --sharedir show location of architecture-independent support files --sysconfdir show location of system-wide configuration files ``` But perhaps also excluding --sysconfdir? > * Next, there should be an extension lookup path. The first item in the path is the compile-time default, and ideally would include only core extensions. Subsequent paths would be set by a GUC, similar to dynamic_library_path, but only for extensions (including their shared libraries). Let’s call it extension_path. I also suggest adding two new pg_config options, for the directory containing core extensions, and a second for system or user extensions. Something like: --extension-dir show location of core extensions --extension-dir-user show location of user extensions The default value for the `extension_path` GUC would be, assuming some new template variables: extension_path = '$userextdir,$extdir' This will allow installers (PGXS) to know where to install non-core extensions without bothering the user about it. > * Modify PGXS (or create a new installer CLI used by PGXS?) to install an extension according to this pattern. Allow the specification of a prefix. This should differ from the current `PREFIX`, in that the values of `sharedir`, `pkglibdir`, etc. would not be fully-duplicated under the prefix, but point to a directory used in the extension path. For example, when installing an extension need “pair", something like > > make install BASE_DIR=/opt/pg/extension > > Would create `/opt/pg/extension/pair`, rather than `/opt/pg/extension/$(pg_config --sharedir)/extension/pair`. > > * Perhaps there could also be an option to symlink binary files or man pages to keep paths simple. > > * For CREATE EXTENSION, Postgres would look for an extension on the file system by directory name in each of the extension paths instead of control file name. It would then find the control file in that directory and the necessary SQL and shared library files in the `sql` and `lib` subdirectories of that directory. In discussion, I think we clarified that it should look for $extension/$extension.control. > * Binary-only extensions might also be installed here; the difference is they have no control file. The LOAD command and shared_preload_libraries would need to know to look here, too. Or perhaps we should require a control file for these, too, but add a “type” key or some such? Maybe such a shared module could be supported by CREATE EXTENSION, as well as, but not include SQL files? > The basic idea, then, is three-fold: > > 1. This pattern is more like a packaging pattern than CREATE EXTENSION-specific, since it includes other types of extensions > > 2. All the files for a given extension live within a single directory, making it easier to reason about what’s installed and what’s not. > > 3. These extension packages can live in multiple paths. For dupes, the first one found in the list of extension_path directories is the one that Postgres will load. We also discussed including the version in the directory name, so that multiple versions could be installed at once. Not sure how Postgres would pick the right one, though. > Some examples. Core extensions, like citext, would live in, say, $(pg_config --extensiondir)/citext), and have a structure such as: > > ``` > citext > ├── citext.control > ├── lib > │ ├── citext.dylib > │ └── bitcode > │ ├── citext > │ │ └── citext.bc > │ └── citext.index.bc > └── sql > ├── citext--1.0--1.1.sql > ├── citext--1.1--1.2.sql > ├── citext--1.2--1.3.sql > ├── citext--1.3--1.4.sql > ├── citext--1.4--1.5.sql > ├── citext--1.4.sql > └── citext--1.5--1.6.sql > ``` > > Third-party extensions would live in one or more other directories on the file system, unknown at compile time, but set in the extension path GUC and accessible to/owned by the Postgres system user. Let’s say we set `/opt/pgxn` as one of the paths. Within that directory, we might have a directory for a pure SQL extension in a a directory named “pair” that looks like this: > > ``` > pair > ├── LICENSE.md > ├── README.md > ├── pair.control > ├── doc > │ ├── html > │ │ └── pair.html > │ └── pair.md > └── sql > ├── pair--1.0--1.1.sql > └── pair--1.1.sql > ``` > > A binary application like pg_top would live in the pg_top directory, structured something like: > > ``` > pg_top > ├── HISTORY.rst > ├── INSTALL.rst > ├── LICENSE > ├── README.rst > ├── bin > | └── pg_top > └── doc > └── man > └── man3 > └── pg_top.3 > ``` > > And a C extension like semver would live in the semver directory and be structured something like: > > ``` > semver > ├── LICENSE > ├── README.md > ├── semver.control > ├── doc > │ └── semver.md > ├── lib > │ ├── semver.dylib > │ ├── bitcode > │ └── semver > │ │ └── semver.bc > │ └── semver.index.bc > └── sql > ├── semver--1.0--1.1.sql > └── semver--1.1.sql > ``` Another example: a binary-only extension loaded via LOAD (or *_preload_libraries), and not `CREATE EXTENSION`, like auto_explain: ``` auto_explain ├── auto_explain.control └── lib ├── auto_explain.dylib ├── bitcode └── auto_explain │ └── auto_explain.bc └── auto_explain.index.bc ``` I’ve included the control file, as suggested above, as a way to manage *all* extensions, not just `CREATE EXTENSION` extensions. A non-core extension would be the same, but might include other files like a README, LICENSE, etc. One wrinkle: Some extensions, such as pg_hint_plan[2], include both a `CREATE EXTENSION` extension and a `LOAD` module, and both have the same name. Not sure how best to adapt for such a case to the proposal to include a control file for both types of extension --- because both would have the same control file name. My proposal is that names would be unique between both `CREATE EXTENSION` and `LOAD` extensions, in which case one or the other extension would need to be renamed (probably the `LOAD` extension). Then perhaps the `CREATE EXTENSION` extension could declare the `LOAD` extension as a dependency. ## Use Cases Here’s how the proposed file layout and extension_path feature would work for the use cases that have driven it. ### Apt/Yum testing Rather than patching Postgres to look up pg_config directories under a prefix[3], a packager who wants to run tests and therefore needs to install an extension where Postgres can find it without writing to the installed server would follow these steps: * Set the extension_path GUC to search the package DESTDIR. * Install the extension into that directory: `make install BASE_DIR=$DESTDIR` * Run `make installcheck` This should allow Postgres to find and load the extension during the tests. The Postgres installation will not have been modified, only the extension_path will have been changed. ### Postgres.app The contents of the macOS Postgres.app bundle must be immutable in order to validate against the signature generated by an Apple-provided certificate. In order to allow extensions to be installed without changing the app bundle, the app would either: 1. Ship with an extension_path pointing to a directory outside the bundle, and then users have to know what this directory is when `make install`ing an extension; or 2. Ship with the --extension-dir-user pg_config value described above pointing to a directory outside the bundle, into which all non-core extensions would be `make install`ed into. ### Docker/Kubernetes Like Postgres.app, Docker images are immutable, but unlike Postgres.app, they represent the entire system. The solution is identical to that for Postgres.app, except that instead of installing extensions into --extension-dir-user, they would be mounted as volumes in that directory. So, instead of `make install`ing there, a Kubernetes pod can be configured to mount a volume for each extension. Need to add a new extension to a Pod? Just mount a volume for it that contains the necessary files, as described in the examples above. One wrinkle: Some Kubernetes providers limit the number of volumes that can be mounted[4]. If someone needed more volumes than that, one would need to adopt a different pattern. Perhaps there could be one volume for the extension-dir-user, and an external service could add any and all necessary extensions to it? ## Challenge: Third Party Dependencies Some extensions require third-party dependencies, usually provided by the OS. For example, pgsql-http[5] compiles into a DSO that dynamically requires another DSO, libcurl. Finding and loading of such dependencies is handled by the OS, not by Postgres. This configuration is an annoyance on most systems, where the user has to figure out what dependencies to install from their OS package manager in order to get it to work. However, it presents a greater challenge for immutable conditions, as in Docker and Kubernetes containers. How can system dependencies be added without breaking immutability? There are a few options: 1. Just include all likely dependencies in the base image. This works today, but it would be preferable not to include additional dependencies that may never be used. It also can unnecessarily bloat an image. Also just kinda gross. 2. Mount a volume with all of the default base image DSOs, then have an external process add DSOs to it when required for a new extension. This also might work today, although it requires coding that external process (e.g., a Kubernetes operator). 3. Mount a second volume for non-base image DSOs, and again have an external process put them there when needed, but then use some method to tell the OS where to find them, since they won’t be in the default location. More on that below. 4. Mount individual DSO files as volumes[6] as needed in the system shared lib directory where the OS can find them. The wrinkle here is the mount limitation imposed by some providers, detailed above. For solutions that require installing a DSO outside the default directories that the system is aware of, one needs a way to tell the system where to find them. There are two basic methods for doing so: 1. Set LD_LIBRARY_PATH to the directory in which third-party DSOs are installed. Today, however, this is considered an insecure pattern[7]. It doesn’t work at all on macOS, for example, unless you disable SIP[8]. Few will do so, nor should they. Andres Freund reports that it’s on its way out on Linux, too. So perhaps some can do this, but it sounds as if LD_LIBRARY_PATH’s days are numbered. 2. Use `-rpath` when compiling the DSOs to point to the proper place. This embeds the path in the DSO, so it always looks in the same place. However, this requires that the DSO be recompiled for every variant of the `-rpath`, which creates challenges for non-path specific binary packaging --- or if an OS vendor changes the director. But perhaps Postgres itself could be compiled with an `-rpath` that will be used when loading extensions, so the extension DSOs themselves don’t have to know the path? Then the base image just needs to be compiled with that option. ## Comments and Corrections I think I captured most of the issues we discussed at PGConf.eu <http://pgconf.eu/; last week; please correct any misunderstandings, inaccuracies, and oversights you spot! And are there any other issues you can think of with the overall approach? Thanks for reading to the end! Best, David [1]: https://www.postgresql.org/message-id/D30A91FA-A6D4-4737-941F-0BBB2984B730%40justatheory.com [2]: https://github.com/ossc-db/pg_hint_plan/ [3]: https://commitfest.postgresql.org/50/4913/ [4]: https://superuser.com/a/1603150/285886 [5]: https://github.com/pramsey/pgsql-http [6]: https://stackoverflow.com/a/42260979/79202 [7]: http://xahlee.info/UnixResource_dir/_/ldpath.html [8]: https://developer.apple.com/documentation/security/disabling-and-enabling-system-integrity-protectio... ^ permalink raw reply [nested|flat] 8+ messages in thread
* Re: RFC: Extension Packaging & Lookup @ 2024-10-29 16:23 Paul Ramsey <[email protected]> parent: David E. Wheeler <[email protected]> 1 sibling, 2 replies; 8+ messages in thread From: Paul Ramsey @ 2024-10-29 16:23 UTC (permalink / raw) To: David E. Wheeler <[email protected]>; +Cc: pgsql-hackers; Gabriele Bartolini <[email protected]>; Peter Eisentraut <[email protected]>; Christoph Berg <[email protected]>; Andres Freund <[email protected]> Thanks for this, David, > On Oct 28, 2024, at 3:19 PM, David E. Wheeler <[email protected]> wrote: > > ## Challenge: Third Party Dependencies This of course is the area that worries the heck out of me, as someone with extensions that includes not just single system dependencies but long chains of them (depending on GDAL draws in a huge tree). > > For solutions that require installing a DSO outside the default directories that the system is aware of, one needs a way to tell the system where to find them. There are two basic methods for doing so: > > 1. Set LD_LIBRARY_PATH to the directory in which third-party DSOs are installed. Today, however, this is considered an insecure pattern[7]. It doesn’t work at all on macOS, for example, unless you disable SIP[8]. Few will do so, nor should they. Andres Freund reports that it’s on its way out on Linux, too. So perhaps some can do this, but it sounds as if LD_LIBRARY_PATH’s days are numbered. > > 2. Use `-rpath` when compiling the DSOs to point to the proper place. This embeds the path in the DSO, so it always looks in the same place. However, this requires that the DSO be recompiled for every variant of the `-rpath`, which creates challenges for non-path specific binary packaging --- or if an OS vendor changes the director. But perhaps Postgres itself could be compiled with an `-rpath` that will be used when loading extensions, so the extension DSOs themselves don’t have to know the path? Then the base image just needs to be compiled with that option. I’m unsure if it will work, but I have wondered if building out the dependencies to install right next to the DSO, and giving the DSO an rpath of “.” would achieve the effect we are looking for. It’s unfortunate (DY)LD_LIBRARY_PATH is dead and dying, but there we are. The trouble I see with somehow coercing the system to load a local copy of system libraries is for (a) common system libs that PostgreSQL itself might be linking (libssl, for example) that then will end up with symbol collisions between the copy loaded by postgres and the copy loaded by the DSO and (b) same thing but for different extensions with the same dependencies. I guess I cannot shake the idea that a lot of interesting extensions are going to have interesting system dependencies, that “exposing an interesting library to postgres” has a high value for an integration system like PostgreSQL. Question for the more knowledgable, how are binary distribution systems like Conda and others shipping DLLs such that different packages don’t clobber each other? P. > > ## Comments and Corrections > > I think I captured most of the issues we discussed at PGConf.eu <http://pgconf.eu/; last week; please correct any misunderstandings, inaccuracies, and oversights you spot! And are there any other issues you can think of with the overall approach? > > Thanks for reading to the end! > > Best, > > David > > [1]: https://www.postgresql.org/message-id/D30A91FA-A6D4-4737-941F-0BBB2984B730%40justatheory.com > [2]: https://github.com/ossc-db/pg_hint_plan/ > [3]: https://commitfest.postgresql.org/50/4913/ > [4]: https://superuser.com/a/1603150/285886 > [5]: https://github.com/pramsey/pgsql-http > [6]: https://stackoverflow.com/a/42260979/79202 > [7]: http://xahlee.info/UnixResource_dir/_/ldpath.html > [8]: https://developer.apple.com/documentation/security/disabling-and-enabling-system-integrity-protectio... > > > ^ permalink raw reply [nested|flat] 8+ messages in thread
* Re: RFC: Extension Packaging & Lookup @ 2024-10-29 16:51 Christoph Berg <[email protected]> parent: Paul Ramsey <[email protected]> 1 sibling, 0 replies; 8+ messages in thread From: Christoph Berg @ 2024-10-29 16:51 UTC (permalink / raw) To: Paul Ramsey <[email protected]>; +Cc: David E. Wheeler <[email protected]>; pgsql-hackers; Gabriele Bartolini <[email protected]>; Peter Eisentraut <[email protected]>; Andres Freund <[email protected]> Re: Paul Ramsey > Thanks for this, David, > > > On Oct 28, 2024, at 3:19 PM, David E. Wheeler <[email protected]> wrote: > > > > ## Challenge: Third Party Dependencies > > This of course is the area that worries the heck out of me, as someone with extensions that includes not just single system dependencies but long chains of them (depending on GDAL draws in a huge tree). I think this is where the whole idea of "provide binaries outside of deb/rpm" is just going to die. You are trying to reinvent a wheel that has been running well for decades, including lots of production systems. I don't know anyone who would trust that new source of binaries that doesn't integrate into their OS packaging system. Christoph ^ permalink raw reply [nested|flat] 8+ messages in thread
* Re: RFC: Extension Packaging & Lookup @ 2024-10-29 17:16 David E. Wheeler <[email protected]> parent: Paul Ramsey <[email protected]> 1 sibling, 0 replies; 8+ messages in thread From: David E. Wheeler @ 2024-10-29 17:16 UTC (permalink / raw) To: Paul Ramsey <[email protected]>; +Cc: pgsql-hackers; Gabriele Bartolini <[email protected]>; Peter Eisentraut <[email protected]>; Christoph Berg <[email protected]>; Andres Freund <[email protected]> On Oct 29, 2024, at 12:23, Paul Ramsey <[email protected]> wrote: > Question for the more knowledgable, how are binary distribution systems like Conda and others shipping DLLs such that different packages don’t clobber each other? I’m not familiar with Conda, but from its docs[1], it seems to rely on a value compiled into an app: > * On Linux, the $ORIGIN variable allows you to specify "relative to this file as it is being executed". > * On macOS, the variables are: > * @rpath---Allows you to set relative links from the system load paths. > * @loader_path---Equivalent to $ORIGIN. > * @executable_path---Supports the Apple .app directory approach, where libraries know where they live relative to their calling application. Thinks are a bit more complicated on Windows, which doesn’t support something like -rpath. D [1]: https://docs.conda.io/projects/conda-build/en/latest/resources/use-shared-libraries.html ^ permalink raw reply [nested|flat] 8+ messages in thread
* Re: RFC: Extension Packaging & Lookup @ 2024-10-31 19:41 David E. Wheeler <[email protected]> parent: David E. Wheeler <[email protected]> 1 sibling, 1 reply; 8+ messages in thread From: David E. Wheeler @ 2024-10-31 19:41 UTC (permalink / raw) To: pgsql-hackers; +Cc: Gabriele Bartolini <[email protected]>; Peter Eisentraut <[email protected]>; Christoph Berg <[email protected]>; Andres Freund <[email protected]> Fellow Humans, I’m working on an updated proposal with more detail, and more comprehensive. But I keep getting a bit caught up on this bit: On Oct 28, 2024, at 18:19, David E. Wheeler <[email protected]> wrote: >> * Binary-only extensions might also be installed here; the difference is they have no control file. The LOAD command and shared_preload_libraries would need to know to look here, too. > > Or perhaps we should require a control file for these, too, but add a “type” key or some such? Maybe such a shared module could be supported by CREATE EXTENSION, as well as, but not include SQL files? I’m trying to imagine how this ought to work. The challenge is that, with the layout I propose here, shared module files will no longer always be in `$dynamic_library_path`, but in any `$extension/pkglib` subdirectory of each subdirectory of `extension_path`, as well. Is that desirable? Let’s say we want to load a module named “semver” that’s included in the semver extension. With the proposed organization up-thread, the module would be installed in: ``` $extdir_user/semver/pkglib/semver.(so|dylib|dll|etc) ``` What should be passed to preload/LOAD to load it? A few options: Option 1 -------- * Specify the module name “semver” in the `LOAD` command or in `*_preload_libraries` (same as in 17 and earlier) * Teach the preload/LOAD code to search for the module file in `*/pkglib/` under each extension path Pros: * Follows the existing module name specification in preload/LOAD Cons: * Potentially huge number of directories to search, when lots of extension are installed. * Depending on search order, the wrong module may be loaded if two extensions have a module file with the same name Option 2 -------- * Specify the module name to include the extension name. Perhaps something like `$extension:$module`. * Teach the preload/LOAD code to detect the extension name as part of the command and only look for the DSO in that extension's `pkglibdir`. Pros: * Searches at most the list of directories in the `extension_path`. * No conflicts with any other module files from other extensions. Cons: * Overloads the meaning of preload/LOAD strings, which might be confusing to some. * Upgrades might need these values to change from the old to the new syntax. Other Options? -------------- I kind of like Option 2, as it would allow us to eventually support non-`CREATE EXTENSION` modules as extensions, too. I imagine distributing, say `auto_explain` in an extension directory of its own, with a `auto_explain.control` file that identifies it as a LOAD-only extension. Then specifying `auto_explain:auto_explain` would work as expected. Or perhaps just `auto_explain:` could load *all* the modules included in the auto_explain "extension". But then maybe I'm starting to talk myself into arguing that `LOAD` ought to be deprecated, `CREATE EXTENSION` could support non-SQL extensions, and the `*preload*` GUCs would contain a list of extensions rather than module files. But I digress. Any ideas about other options to address this design challenge? Thanks, David ^ permalink raw reply [nested|flat] 8+ messages in thread
* Re: RFC: Extension Packaging & Lookup @ 2024-10-31 20:12 David E. Wheeler <[email protected]> parent: David E. Wheeler <[email protected]> 0 siblings, 1 reply; 8+ messages in thread From: David E. Wheeler @ 2024-10-31 20:12 UTC (permalink / raw) To: pgsql-hackers; +Cc: Gabriele Bartolini <[email protected]>; Peter Eisentraut <[email protected]>; Christoph Berg <[email protected]>; Andres Freund <[email protected]> On Oct 31, 2024, at 15:41, David E. Wheeler <[email protected]> wrote: > Other Options? > -------------- > > I kind of like Option 2, as it would allow us to eventually support non-`CREATE EXTENSION` modules as extensions, too. I imagine distributing, say `auto_explain` in an extension directory of its own, with a `auto_explain.control` file that identifies it as a LOAD-only extension. Then specifying `auto_explain:auto_explain` would work as expected. Or perhaps just `auto_explain:` could load *all* the modules included in the auto_explain "extension". > > But then maybe I'm starting to talk myself into arguing that `LOAD` ought to be deprecated, `CREATE EXTENSION` could support non-SQL extensions, and the `*preload*` GUCs would contain a list of extensions rather than module files. > > But I digress. Any ideas about other options to address this design challenge? I just thought of another one: Option 3 -------- * Add a new preload GUCs for extensions: `shared_preload_extensions`, `session_preload_extensions`, etc. * Specify just extension names for these GUCs, instead of module file names. * Leave `LOAD/*preload_libraries` unchanged. * Have it search the `extension_path` directories just as `CREATE EXTENSION` does. Pros: * The behaviors of `LOAD/*preload_libraries` are unchanged * Provide a more future-looking interface, where we perhaps eventually deprecate `LOAD/*preload_libraries` in favor of shipping all modules as extensions. Cons: * More GUCs! * The new GUCs load *all* of the modules included in an extension, rather than specific ones. But maybe we can borrow the `$extension:module` syntax from Option 2 to support loading a single extension * Upgrades from Postgres 17 would still need any extension modules loaded in `*preload_libraries` moved to the new GUCs, since the files will live in a new location I kind of like this one… D ^ permalink raw reply [nested|flat] 8+ messages in thread
* Re: RFC: Extension Packaging & Lookup @ 2024-11-07 15:30 David E. Wheeler <[email protected]> parent: David E. Wheeler <[email protected]> 0 siblings, 0 replies; 8+ messages in thread From: David E. Wheeler @ 2024-11-07 15:30 UTC (permalink / raw) To: pgsql-hackers; +Cc: Gabriele Bartolini <[email protected]>; Peter Eisentraut <[email protected]>; Christoph Berg <[email protected]>; Andres Freund <[email protected]> Hackers, On Oct 31, 2024, at 16:12, David E. Wheeler <[email protected]> wrote: > I just thought of another one: Last week I tried to integrate all the ideas into this thread and the previous[1] into a single proposal that attempts to work through all the implications and issues. I’ve drafted it as a blog post[2] and plan to publish it next week, following some more feedback. Would appreciate comments, corrections, and any other general feedback: https://github.com/theory/justatheory/pull/7 Best, David [1]: https://postgr.es/m/[email protected] [2]: https://github.com/theory/justatheory/pull/7 ^ permalink raw reply [nested|flat] 8+ messages in thread
end of thread, other threads:[~2024-11-07 15:30 UTC | newest] Thread overview: 8+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-11-20 02:48 [PATCH 1/2] bootstrap: convert Typ to a List* Justin Pryzby <[email protected]> 2024-10-28 22:19 Re: RFC: Extension Packaging & Lookup David E. Wheeler <[email protected]> 2024-10-29 16:23 ` Re: RFC: Extension Packaging & Lookup Paul Ramsey <[email protected]> 2024-10-29 16:51 ` Re: RFC: Extension Packaging & Lookup Christoph Berg <[email protected]> 2024-10-29 17:16 ` Re: RFC: Extension Packaging & Lookup David E. Wheeler <[email protected]> 2024-10-31 19:41 ` Re: RFC: Extension Packaging & Lookup David E. Wheeler <[email protected]> 2024-10-31 20:12 ` Re: RFC: Extension Packaging & Lookup David E. Wheeler <[email protected]> 2024-11-07 15:30 ` Re: RFC: Extension Packaging & Lookup David E. Wheeler <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox