public inbox for [email protected]help / color / mirror / Atom feed
[PATCH v24 03/10] Add default_toast_compression GUC 3+ messages / 3 participants [nested] [flat]
* [PATCH v24 03/10] Add default_toast_compression GUC @ 2021-01-30 03:37 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Justin Pryzby @ 2021-01-30 03:37 UTC (permalink / raw) --- src/backend/access/common/tupdesc.c | 2 +- src/backend/bootstrap/bootstrap.c | 3 +- src/backend/commands/amcmds.c | 143 +++++++++++++++++++++++++++- src/backend/commands/tablecmds.c | 4 +- src/backend/utils/init/postinit.c | 4 + src/backend/utils/misc/guc.c | 12 +++ src/include/access/amapi.h | 2 + src/include/access/compressamapi.h | 12 ++- 8 files changed, 176 insertions(+), 6 deletions(-) diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index ca26fab487..7afaea000b 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -668,7 +668,7 @@ TupleDescInitEntry(TupleDesc desc, att->attcollation = typeForm->typcollation; if (IsStorageCompressible(typeForm->typstorage)) - att->attcompression = DefaultCompressionOid; + att->attcompression = GetDefaultToastCompression(); else att->attcompression = InvalidOid; diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 9b451eaa71..ec3376cf8a 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -732,8 +732,9 @@ DefineAttr(char *name, char *type, int attnum, int nullness) attrtypes[attnum]->attcacheoff = -1; attrtypes[attnum]->atttypmod = -1; attrtypes[attnum]->attislocal = true; + if (IsStorageCompressible(attrtypes[attnum]->attstorage)) - attrtypes[attnum]->attcompression = DefaultCompressionOid; + attrtypes[attnum]->attcompression = GetDefaultToastCompression(); else attrtypes[attnum]->attcompression = InvalidOid; diff --git a/src/backend/commands/amcmds.c b/src/backend/commands/amcmds.c index 3ad4a61739..1682afd2a4 100644 --- a/src/backend/commands/amcmds.c +++ b/src/backend/commands/amcmds.c @@ -13,8 +13,11 @@ */ #include "postgres.h" +#include "access/amapi.h" +#include "access/compressamapi.h" #include "access/htup_details.h" #include "access/table.h" +#include "access/xact.h" #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/indexing.h" @@ -27,13 +30,20 @@ #include "parser/parse_func.h" #include "utils/builtins.h" #include "utils/lsyscache.h" +#include "utils/guc.h" +#include "utils/inval.h" #include "utils/rel.h" #include "utils/syscache.h" - static Oid lookup_am_handler_func(List *handler_name, char amtype); static const char *get_am_type_string(char amtype); +/* Compile-time default */ +char *default_toast_compression = DEFAULT_TOAST_COMPRESSION; // maybe needs pg_dump support ? +/* Invalid means need to lookup the text value in the catalog */ +static Oid default_toast_compression_oid = InvalidOid; + +static void AccessMethodCallback(Datum arg, int cacheid, uint32 hashvalue); /* * CreateAccessMethod @@ -277,3 +287,134 @@ lookup_am_handler_func(List *handler_name, char amtype) return handlerOid; } + +/* check_hook: validate new default_toast_compression */ +bool +check_default_toast_compression(char **newval, void **extra, GucSource source) +{ + if (**newval == '\0') + { + GUC_check_errdetail("%s cannot be empty.", + "default_toast_compression"); + return false; + } + + if (strlen(*newval) >= NAMEDATALEN) + { + GUC_check_errdetail("%s is too long (maximum %d characters).", + "default_toast_compression", NAMEDATALEN - 1); + return false; + } + + /* + * If we aren't inside a transaction, or not connected to a database, we + * cannot do the catalog access necessary to verify the method. Must + * accept the value on faith. + */ + if (IsTransactionState() && MyDatabaseId != InvalidOid) + { + if (!OidIsValid(get_compression_am_oid(*newval, true))) + { + /* + * When source == PGC_S_TEST, don't throw a hard error for a + * nonexistent table access method, only a NOTICE. See comments in + * guc.h. + */ + if (source == PGC_S_TEST) + { + ereport(NOTICE, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("compression access method \"%s\" does not exist", + *newval))); + } + else + { + GUC_check_errdetail("Compression access method \"%s\" does not exist.", + *newval); + return false; + } + } + } + + return true; +} + +/* + * assign_default_toast_compression: GUC assign_hook for default_toast_compression + */ +void +assign_default_toast_compression(const char *newval, void *extra) +{ + /* + * Invalidate setting, forcing it to be looked up as needed. + * This avoids trying to do database access during GUC initialization, + * or outside a transaction. + */ + + default_toast_compression = NULL; +fprintf(stderr, "set to null\n"); +} + + +/* + * GetDefaultToastCompression -- get the OID of the current toast compression + * + * This exists to hide and optimize the use of the default_toast_compression + * GUC variable. + */ +Oid +GetDefaultToastCompression(void) +{ + /* Avoid catalog access during bootstrap, and for default compression */ + if (strcmp(default_toast_compression, DEFAULT_TOAST_COMPRESSION) == 0) + return PGLZ_COMPRESSION_AM_OID; + + /* Cannot call get_compression_am_oid this early */ + // if (IsBootstrapProcessingMode()) + // return PGLZ_COMPRESSION_AM_OID; + Assert(!IsBootstrapProcessingMode()); + + /* + * If cached value isn't valid, look up the current default value, caching + * the result + */ + if (!OidIsValid(default_toast_compression_oid)) + default_toast_compression_oid = + get_am_type_oid(default_toast_compression, AMTYPE_COMPRESSION, + false); + + return default_toast_compression_oid; +} + +/* + * InitializeAccessMethods: initialize module during InitPostgres. + * + * This is called after we are up enough to be able to do catalog lookups. + */ +void +InitializeAccessMethods(void) +{ + if (IsBootstrapProcessingMode()) + return; + + /* + * In normal mode, arrange for a callback on any syscache invalidation + * of pg_am rows. + */ + CacheRegisterSyscacheCallback(AMOID, + AccessMethodCallback, + (Datum) 0); + /* Force cached default access method to be recomputed on next use */ + // default_toast_compression_oid = InvalidOid; +} + +/* + * AccessMethodCallback + * Syscache inval callback function + */ +static void +AccessMethodCallback(Datum arg, int cacheid, uint32 hashvalue) +{ + /* Force look up of compression oid on next use */ + default_toast_compression_oid = false; +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index dd81d5bf4e..72ba017814 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -11925,7 +11925,7 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, if (!IsStorageCompressible(tform->typstorage)) attTup->attcompression = InvalidOid; else if (!OidIsValid(attTup->attcompression)) - attTup->attcompression = DefaultCompressionOid; + attTup->attcompression = GetDefaultToastCompression(); } else attTup->attcompression = InvalidOid; @@ -17762,7 +17762,7 @@ GetAttributeCompression(Form_pg_attribute att, char *compression) /* fallback to default compression if it's not specified */ if (compression == NULL) - return DefaultCompressionOid; + return GetDefaultToastCompression(); amoid = get_compression_am_oid(compression, false); diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index e5965bc517..6a02bbd377 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -25,6 +25,7 @@ #include "access/session.h" #include "access/sysattr.h" #include "access/tableam.h" +#include "access/amapi.h" #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" @@ -1060,6 +1061,9 @@ InitPostgres(const char *in_dbname, Oid dboid, const char *username, /* set default namespace search path */ InitializeSearchPath(); + /* set callback for changes to pg_am */ + InitializeAccessMethods(); + /* initialize client encoding */ InitializeClientEncoding(); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index eafdb1118e..3a2b33fcd1 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -30,6 +30,7 @@ #include <unistd.h> #include "access/commit_ts.h" +#include "access/compressamapi.h" #include "access/gin.h" #include "access/rmgr.h" #include "access/tableam.h" @@ -3915,6 +3916,17 @@ static struct config_string ConfigureNamesString[] = check_default_table_access_method, NULL, NULL }, + { + {"default_toast_compression", PGC_USERSET, CLIENT_CONN_STATEMENT, + gettext_noop("Sets the default compression for new columns."), + NULL, + GUC_IS_NAME + }, + &default_toast_compression, + DEFAULT_TOAST_COMPRESSION, + check_default_toast_compression, assign_default_toast_compression, NULL, NULL + }, + { {"default_tablespace", PGC_USERSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets the default tablespace to create tables and indexes in."), diff --git a/src/include/access/amapi.h b/src/include/access/amapi.h index d357ebb559..1513cafcf4 100644 --- a/src/include/access/amapi.h +++ b/src/include/access/amapi.h @@ -287,4 +287,6 @@ typedef struct IndexAmRoutine extern IndexAmRoutine *GetIndexAmRoutine(Oid amhandler); extern IndexAmRoutine *GetIndexAmRoutineByAmId(Oid amoid, bool noerror); +void InitializeAccessMethods(void); + #endif /* AMAPI_H */ diff --git a/src/include/access/compressamapi.h b/src/include/access/compressamapi.h index 5a8e23d926..d75a8e9df2 100644 --- a/src/include/access/compressamapi.h +++ b/src/include/access/compressamapi.h @@ -17,6 +17,7 @@ #include "catalog/pg_am_d.h" #include "nodes/nodes.h" +#include "utils/guc.h" /* * Built-in compression method-id. The toast compression header will store @@ -29,8 +30,17 @@ typedef enum CompressionId LZ4_COMPRESSION_ID = 1 } CompressionId; -/* Use default compression method if it is not specified. */ +/* Default compression method if not specified. */ +#define DEFAULT_TOAST_COMPRESSION "pglz" #define DefaultCompressionOid PGLZ_COMPRESSION_AM_OID + +/* GUC */ +extern char *default_toast_compression; +extern void assign_default_toast_compression(const char *newval, void *extra); +extern bool check_default_toast_compression(char **newval, void **extra, GucSource source); + +extern Oid GetDefaultToastCompression(void); + #define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \ (storage) != TYPSTORAGE_EXTERNAL) /* compression handler routines */ -- 2.17.0 --m51xatjYGsM+13rf Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="v24-0004-default-to-with-lz4.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* UUID v7 @ 2023-02-10 23:57 Andrey Borodin <[email protected]> 0 siblings, 1 reply; 3+ messages in thread From: Andrey Borodin @ 2023-02-10 23:57 UTC (permalink / raw) To: pgsql-hackers; +Cc: [email protected]; [email protected]; [email protected]; Nikolay Samokhvalov <[email protected]> Hello pgsql-hackers! As you may know there's a new version of UUID being standardized [0]. These new algorithms of UUID generation are very promising for database performance. It keeps data locality for time-ordered values. From my POV v7 is especially needed for users. Current standard status is "draft". And I'm not sure it will be accepted before our feature freeze for PG16. Maybe we could provide a draft implementation in 16 and adjust it to the accepted version if the standard is changed? PFA patch with implementation. What do you think? cc Brad Peabody and Kyzer R. Davis, authors of the standard cc Kirk Wolak and Nik Samokhvalov who requested the feature Thanks! Best regards, Andrey Borodin. [0] https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04 Attachments: [application/octet-stream] v1-0001-Implement-UUID-v7-as-per-IETF-draft.patch (5.4K, ../../CAAhFRxitJv=yoGnXUgeLB_O+M7J2BJAmb5jqAT9gZ3bij3uLDA@mail.gmail.com/2-v1-0001-Implement-UUID-v7-as-per-IETF-draft.patch) download | inline diff: From 044008eef6b72f72c07f40bedfecba05d9448855 Mon Sep 17 00:00:00 2001 From: Andrey Borodin <[email protected]> Date: Fri, 10 Feb 2023 15:38:40 -0800 Subject: [PATCH v1] Implement UUID v7 as per IETF draft --- doc/src/sgml/func.sgml | 15 +++++++++- src/backend/utils/adt/uuid.c | 38 ++++++++++++++++++++++++ src/include/catalog/pg_proc.dat | 3 ++ src/test/regress/expected/opr_sanity.out | 1 + src/test/regress/expected/uuid.out | 10 +++++++ src/test/regress/sql/uuid.sql | 6 ++++ 6 files changed, 72 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index e09e289a43..dd6a7914e4 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -13858,14 +13858,27 @@ CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple <primary>gen_random_uuid</primary> </indexterm> + <indexterm> + <primary>gen_uuid_v7</primary> + </indexterm> + + <para> + <productname>PostgreSQL</productname> includes two functions to generate a UUID: + </para> <para> - <productname>PostgreSQL</productname> includes one function to generate a UUID: <synopsis> <function>gen_random_uuid</function> () <returnvalue>uuid</returnvalue> </synopsis> This function returns a version 4 (random) UUID. This is the most commonly used type of UUID and is appropriate for most applications. </para> + <para> + <productname>PostgreSQL</productname> includes two functions to generate a UUID: +<synopsis> +<function>gen_uuid_v7</function> () <returnvalue>uuid</returnvalue> +</synopsis> + This function returns a version 7 (time-ordered + random) UUID. + </para> <para> The <xref linkend="uuid-ossp"/> module provides additional functions that diff --git a/src/backend/utils/adt/uuid.c b/src/backend/utils/adt/uuid.c index 76ff6c533f..66eb4a35de 100644 --- a/src/backend/utils/adt/uuid.c +++ b/src/backend/utils/adt/uuid.c @@ -13,6 +13,8 @@ #include "postgres.h" +#include <sys/time.h> + #include "common/hashfn.h" #include "lib/hyperloglog.h" #include "libpq/pqformat.h" @@ -421,3 +423,39 @@ gen_random_uuid(PG_FUNCTION_ARGS) PG_RETURN_UUID_P(uuid); } + +Datum +gen_uuid_v7(PG_FUNCTION_ARGS) +{ + pg_uuid_t *uuid = palloc(UUID_LEN); + struct timeval tp; + uint64_t tms; + + gettimeofday(&tp, NULL); + + tms = ((uint64_t)tp.tv_sec) * 1000; + tms += ((uint64_t)tp.tv_usec) / 1000; + + tms = pg_hton64(tms<<16); + + /* Fill in time part */ + memcpy(&uuid->data[0], &tms, 6); + + /* fill everything after the timestamp with random bytes */ + if (!pg_strong_random(&uuid->data[6], UUID_LEN - 6)) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate random values"))); + + /* + * Set magic numbers for a "version 7" (pseudorandom) UUID, see + * http://tools.ietf.org/html/rfc ??? + * https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format#name-creating-a-uuidv7-value + */ + /* set version field, top four bits are 0, 1, 1, 1 */ + uuid->data[6] = (uuid->data[6] & 0x0f) | 0x70; + /* set variant field, top two bits are 1, 0 */ + uuid->data[8] = (uuid->data[8] & 0x3f) | 0x80; + + PG_RETURN_UUID_P(uuid); +} diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index c0f2a8a77c..d448b0a338 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -9042,6 +9042,9 @@ { oid => '3432', descr => 'generate random UUID', proname => 'gen_random_uuid', proleakproof => 't', provolatile => 'v', prorettype => 'uuid', proargtypes => '', prosrc => 'gen_random_uuid' }, +{ oid => '3813', descr => 'generate UUID version 7', + proname => 'gen_uuid_v7', proleakproof => 't', provolatile => 'v', + prorettype => 'uuid', proargtypes => '', prosrc => 'gen_uuid_v7' }, # pg_lsn { oid => '3229', descr => 'I/O', diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 02f5348ab1..543ea38f43 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -857,6 +857,7 @@ sha384(bytea) sha512(bytea) gen_random_uuid() starts_with(text,text) +gen_uuid_v7() macaddr8_eq(macaddr8,macaddr8) macaddr8_lt(macaddr8,macaddr8) macaddr8_le(macaddr8,macaddr8) diff --git a/src/test/regress/expected/uuid.out b/src/test/regress/expected/uuid.out index 0f47232009..c577732e3e 100644 --- a/src/test/regress/expected/uuid.out +++ b/src/test/regress/expected/uuid.out @@ -168,5 +168,15 @@ SELECT count(DISTINCT guid_field) FROM guid1; 2 (1 row) +-- generation test for v7 +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (gen_uuid_v7()); +INSERT INTO guid1 (guid_field) VALUES (gen_uuid_v7()); +SELECT count(DISTINCT guid_field) FROM guid1; + count +------- + 2 +(1 row) + -- clean up DROP TABLE guid1, guid2 CASCADE; diff --git a/src/test/regress/sql/uuid.sql b/src/test/regress/sql/uuid.sql index 37d954eda1..c54dbf4eb2 100644 --- a/src/test/regress/sql/uuid.sql +++ b/src/test/regress/sql/uuid.sql @@ -85,5 +85,11 @@ INSERT INTO guid1 (guid_field) VALUES (gen_random_uuid()); INSERT INTO guid1 (guid_field) VALUES (gen_random_uuid()); SELECT count(DISTINCT guid_field) FROM guid1; +-- generation test for v7 +TRUNCATE guid1; +INSERT INTO guid1 (guid_field) VALUES (gen_uuid_v7()); +INSERT INTO guid1 (guid_field) VALUES (gen_uuid_v7()); +SELECT count(DISTINCT guid_field) FROM guid1; + -- clean up DROP TABLE guid1, guid2 CASCADE; -- 2.37.0 (Apple Git-136) ^ permalink raw reply [nested|flat] 3+ messages in thread
* Re: UUID v7 @ 2023-02-11 01:14 Andres Freund <[email protected]> parent: Andrey Borodin <[email protected]> 0 siblings, 0 replies; 3+ messages in thread From: Andres Freund @ 2023-02-11 01:14 UTC (permalink / raw) To: Andrey Borodin <[email protected]>; +Cc: pgsql-hackers; [email protected]; [email protected]; [email protected]; Nikolay Samokhvalov <[email protected]> Hi, On 2023-02-10 15:57:50 -0800, Andrey Borodin wrote: > As you may know there's a new version of UUID being standardized [0]. > These new algorithms of UUID generation are very promising for > database performance. I agree it's very useful to have. > [0] https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-04 That looks to not be the current version anymore, it's superseded by: https://datatracker.ietf.org/doc/html/draft-ietf-uuidrev-rfc4122bis > It keeps data locality for time-ordered values. > From my POV v7 is especially needed for users. Current standard status > is "draft". And I'm not sure it will be accepted before our feature > freeze for PG16. Maybe we could provide a draft implementation in 16 > and adjust it to the accepted version if the standard is changed? PFA > patch with implementation. Hm. It seems somewhat worrisome to claim something is a v7 UUID when it might turn out to not be one. Perhaps we should name the function something like gen_time_ordered_random_uuid() instead? That gives us a bit more flexibility about what uuid version we generate. And it might be easier for users, anyway. Still not sure what version we'd best use for now. Perhaps v8? Greetings, Andres Freund ^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2023-02-11 01:14 UTC | newest] Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-01-30 03:37 [PATCH v24 03/10] Add default_toast_compression GUC Justin Pryzby <[email protected]> 2023-02-10 23:57 UUID v7 Andrey Borodin <[email protected]> 2023-02-11 01:14 ` Re: UUID v7 Andres Freund <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox