public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 6/8] Add default_toast_compression GUC
10+ messages / 5 participants
[nested] [flat]
* [PATCH 6/8] Add default_toast_compression GUC
@ 2021-03-10 08:35 Dilip Kumar <[email protected]>
0 siblings, 0 replies; 10+ messages in thread
From: Dilip Kumar @ 2021-03-10 08:35 UTC (permalink / raw)
Justin Pryzby and Dilip Kumar
---
src/backend/access/common/toast_compression.c | 46 +++++++++++++++++++
src/backend/access/common/tupdesc.c | 2 +-
src/backend/bootstrap/bootstrap.c | 2 +-
src/backend/commands/tablecmds.c | 8 ++--
src/backend/utils/misc/guc.c | 12 +++++
src/backend/utils/misc/postgresql.conf.sample | 1 +
src/include/access/toast_compression.h | 22 ++++++++-
src/test/regress/expected/compression.out | 16 +++++++
src/test/regress/expected/compression_1.out | 19 ++++++++
src/test/regress/sql/compression.sql | 8 ++++
10 files changed, 128 insertions(+), 8 deletions(-)
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 3463b42438..e33f92687c 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -55,6 +55,9 @@ const CompressionRoutine toast_compression[] =
}
};
+/* Compile-time default */
+char *default_toast_compression = DEFAULT_TOAST_COMPRESSION;
+
/*
* pglz_cmcompress - compression routine for pglz compression method
*
@@ -306,3 +309,46 @@ GetCompressionRoutines(char method)
{
return &toast_compression[CompressionMethodToId(method)];
}
+
+/* check_hook: validate new default_toast_compression */
+bool
+check_default_toast_compression(char **newval, void **extra, GucSource source)
+{
+ if (**newval == '\0')
+ {
+ GUC_check_errdetail("%s cannot be empty.",
+ "default_toast_compression");
+ return false;
+ }
+
+ if (strlen(*newval) >= NAMEDATALEN)
+ {
+ GUC_check_errdetail("%s is too long (maximum %d characters).",
+ "default_toast_compression", NAMEDATALEN - 1);
+ return false;
+ }
+
+ if (!CompressionMethodIsValid(CompressionNameToMethod(*newval)))
+ {
+ /*
+ * When source == PGC_S_TEST, don't throw a hard error for a
+ * nonexistent compression method, only a NOTICE. See comments in
+ * guc.h.
+ */
+ if (source == PGC_S_TEST)
+ {
+ ereport(NOTICE,
+ (errcode(ERRCODE_UNDEFINED_OBJECT),
+ errmsg("compression method \"%s\" does not exist",
+ *newval)));
+ }
+ else
+ {
+ GUC_check_errdetail("Compression method \"%s\" does not exist.",
+ *newval);
+ return false;
+ }
+ }
+
+ return true;
+}
diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c
index a66d1113db..362a371a6c 100644
--- a/src/backend/access/common/tupdesc.c
+++ b/src/backend/access/common/tupdesc.c
@@ -668,7 +668,7 @@ TupleDescInitEntry(TupleDesc desc,
att->attcollation = typeForm->typcollation;
if (IsStorageCompressible(typeForm->typstorage))
- att->attcompression = DefaultCompressionMethod;
+ att->attcompression = GetDefaultToastCompression();
else
att->attcompression = InvalidCompressionMethod;
diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c
index af0c0aa77e..9be0841d08 100644
--- a/src/backend/bootstrap/bootstrap.c
+++ b/src/backend/bootstrap/bootstrap.c
@@ -733,7 +733,7 @@ DefineAttr(char *name, char *type, int attnum, int nullness)
attrtypes[attnum]->atttypmod = -1;
attrtypes[attnum]->attislocal = true;
if (IsStorageCompressible(attrtypes[attnum]->attstorage))
- attrtypes[attnum]->attcompression = DefaultCompressionMethod;
+ attrtypes[attnum]->attcompression = GetDefaultToastCompression();
else
attrtypes[attnum]->attcompression = InvalidCompressionMethod;
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index e20dd5f63c..a0d2af0cde 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -11971,7 +11971,7 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel,
if (!IsStorageCompressible(tform->typstorage))
attTup->attcompression = InvalidCompressionMethod;
else if (!CompressionMethodIsValid(attTup->attcompression))
- attTup->attcompression = DefaultCompressionMethod;
+ attTup->attcompression = GetDefaultToastCompression();
}
else
attTup->attcompression = InvalidCompressionMethod;
@@ -17847,9 +17847,9 @@ GetAttributeCompression(Form_pg_attribute att, char *compression)
/* fallback to default compression if it's not specified */
if (compression == NULL)
- return DefaultCompressionMethod;
-
- cmethod = CompressionNameToMethod(compression);
+ cmethod = GetDefaultToastCompression();
+ else
+ cmethod = CompressionNameToMethod(compression);
return cmethod;
}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 953837085f..e59ed5b214 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -33,6 +33,7 @@
#include "access/gin.h"
#include "access/rmgr.h"
#include "access/tableam.h"
+#include "access/toast_compression.h"
#include "access/transam.h"
#include "access/twophase.h"
#include "access/xact.h"
@@ -3915,6 +3916,17 @@ static struct config_string ConfigureNamesString[] =
check_default_table_access_method, NULL, NULL
},
+ {
+ {"default_toast_compression", PGC_USERSET, CLIENT_CONN_STATEMENT,
+ gettext_noop("Sets the default compression for new columns."),
+ NULL,
+ GUC_IS_NAME
+ },
+ &default_toast_compression,
+ DEFAULT_TOAST_COMPRESSION,
+ check_default_toast_compression, NULL, NULL
+ },
+
{
{"default_tablespace", PGC_USERSET, CLIENT_CONN_STATEMENT,
gettext_noop("Sets the default tablespace to create tables and indexes in."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index f46c2dd7a8..7d7a433dcc 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -659,6 +659,7 @@
#temp_tablespaces = '' # a list of tablespace names, '' uses
# only default tablespace
#default_table_access_method = 'heap'
+#default_toast_compression = 'pglz' # 'pglz' or 'lz4'
#check_function_bodies = on
#default_transaction_isolation = 'read committed'
#default_transaction_read_only = off
diff --git a/src/include/access/toast_compression.h b/src/include/access/toast_compression.h
index 38800cb97a..e9d9ce5634 100644
--- a/src/include/access/toast_compression.h
+++ b/src/include/access/toast_compression.h
@@ -15,6 +15,14 @@
#include "postgres.h"
+#include "utils/guc.h"
+
+/* default compression method if not specified. */
+#define DEFAULT_TOAST_COMPRESSION "pglz"
+
+/* GUCs */
+extern char *default_toast_compression;
+
/*
* Built-in compression methods. pg_attribute will store this in the
* attcompression column.
@@ -36,8 +44,6 @@ typedef enum CompressionId
LZ4_COMPRESSION_ID = 1
} CompressionId;
-/* use default compression method if it is not specified. */
-#define DefaultCompressionMethod PGLZ_COMPRESSION
#define IsValidCompression(cm) ((cm) != InvalidCompressionMethod)
#define IsStorageCompressible(storage) ((storage) != TYPSTORAGE_PLAIN && \
@@ -67,6 +73,8 @@ typedef struct CompressionRoutine
extern char CompressionNameToMethod(char *compression);
extern const CompressionRoutine *GetCompressionRoutines(char method);
+extern bool check_default_toast_compression(char **newval, void **extra,
+ GucSource source);
/*
* CompressionMethodToId - Convert compression method to compression id.
@@ -115,5 +123,15 @@ GetCompressionMethodName(char method)
return GetCompressionRoutines(method)->cmname;
}
+/*
+ * GetDefaultToastCompression -- get the current toast compression
+ *
+ * This exists to hide the use of the default_toast_compression GUC variable.
+ */
+static inline char
+GetDefaultToastCompression(void)
+{
+ return CompressionNameToMethod(default_toast_compression);
+}
#endif /* TOAST_COMPRESSION_H */
diff --git a/src/test/regress/expected/compression.out b/src/test/regress/expected/compression.out
index 6981d580dc..9ddffc8b18 100644
--- a/src/test/regress/expected/compression.out
+++ b/src/test/regress/expected/compression.out
@@ -231,6 +231,22 @@ CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
NOTICE: merging column "f1" with inherited definition
ERROR: column "f1" has a compression method conflict
DETAIL: pglz versus lz4
+-- test default_toast_compression GUC
+SET default_toast_compression = '';
+ERROR: invalid value for parameter "default_toast_compression": ""
+DETAIL: default_toast_compression cannot be empty.
+SET default_toast_compression = 'I do not exist compression';
+ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
+DETAIL: Compression method "I do not exist compression" does not exist.
+SET default_toast_compression = 'lz4';
+DROP TABLE cmdata2;
+CREATE TABLE cmdata2 (f1 text);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | lz4 | |
+
-- check data is ok
SELECT length(f1) FROM cmdata;
length
diff --git a/src/test/regress/expected/compression_1.out b/src/test/regress/expected/compression_1.out
index 893c18c7fa..03a7f46554 100644
--- a/src/test/regress/expected/compression_1.out
+++ b/src/test/regress/expected/compression_1.out
@@ -237,6 +237,25 @@ CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
NOTICE: merging column "f1" with inherited definition
ERROR: column "f1" has a compression method conflict
DETAIL: pglz versus lz4
+-- test default_toast_compression GUC
+SET default_toast_compression = '';
+ERROR: invalid value for parameter "default_toast_compression": ""
+DETAIL: default_toast_compression cannot be empty.
+SET default_toast_compression = 'I do not exist compression';
+ERROR: invalid value for parameter "default_toast_compression": "I do not exist compression"
+DETAIL: Compression method "I do not exist compression" does not exist.
+SET default_toast_compression = 'lz4';
+ERROR: unsupported LZ4 compression method
+DETAIL: This functionality requires the server to be built with lz4 support.
+HINT: You need to rebuild PostgreSQL using --with-lz4.
+DROP TABLE cmdata2;
+CREATE TABLE cmdata2 (f1 text);
+\d+ cmdata2
+ Table "public.cmdata2"
+ Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description
+--------+------+-----------+----------+---------+----------+-------------+--------------+-------------
+ f1 | text | | | | extended | pglz | |
+
-- check data is ok
SELECT length(f1) FROM cmdata;
length
diff --git a/src/test/regress/sql/compression.sql b/src/test/regress/sql/compression.sql
index b085c9fbd9..6c9b24f1f7 100644
--- a/src/test/regress/sql/compression.sql
+++ b/src/test/regress/sql/compression.sql
@@ -103,6 +103,14 @@ SELECT pg_column_compression(f1) FROM cmpart;
CREATE TABLE cminh() INHERITS(cmdata, cmdata1);
CREATE TABLE cminh(f1 TEXT COMPRESSION lz4) INHERITS(cmdata);
+-- test default_toast_compression GUC
+SET default_toast_compression = '';
+SET default_toast_compression = 'I do not exist compression';
+SET default_toast_compression = 'lz4';
+DROP TABLE cmdata2;
+CREATE TABLE cmdata2 (f1 text);
+\d+ cmdata2
+
-- check data is ok
SELECT length(f1) FROM cmdata;
SELECT length(f1) FROM cmdata1;
--
2.17.0
--cvVnyQ+4j833TQvp
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
filename="0007-Alter-table-set-compression.patch"
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-17 20:05 Joe Conway <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Joe Conway @ 2024-08-17 20:05 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; +Cc: pgsql-hackers
On 8/17/24 13:59, Matthias van de Meent wrote:
> On Sat, 17 Aug 2024 at 18:41, Bruce Momjian <[email protected]> wrote:
>>
>> About 20 minutes ago I starting getting gitmaster pull errors:
>>
>> $ git pull ssh://[email protected]/postgresql.git
>> ssh: connect to host gitmaster.postgresql.org port 22: Connection timed
>> out
>> fatal: Could not read from remote repository.
>>
>> Please make sure you have the correct access rights
>> and the repository exists.
> [...]
>> Can anyone confirm or report on the cause?
>
> Additional infra that doesn't seem to work right now: mailing list
> archives [0] seem to fail at a 503 produced by Varnish Cache server:
> Error 503 Backend fetch failed. Maybe more of infra is down, or
> otherwise under maintenance?
>
> Kind regards,
>
> Matthias van de Meent
>
> [0] https://www.postgresql.org/list/pgsql-hackers/2024-08/
Just to close this out -- problem found and fixed about an hour ago.
Sorry for the interruption.
--
Joe Conway
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 09:04 Jelte Fennema-Nio <[email protected]>
parent: Joe Conway <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Jelte Fennema-Nio @ 2024-08-19 09:04 UTC (permalink / raw)
To: Joe Conway <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On Sat, 17 Aug 2024 at 22:05, Joe Conway <[email protected]> wrote:
> Just to close this out -- problem found and fixed about an hour ago.
> Sorry for the interruption.
Whatever the problem was it seems to have returned
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 11:39 Joe Conway <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Joe Conway @ 2024-08-19 11:39 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On 8/19/24 05:04, Jelte Fennema-Nio wrote:
> On Sat, 17 Aug 2024 at 22:05, Joe Conway <[email protected]> wrote:
>> Just to close this out -- problem found and fixed about an hour ago.
>> Sorry for the interruption.
>
> Whatever the problem was it seems to have returned
What specifically? I am not seeing anything at the moment.
--
Joe Conway
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 11:57 Jelte Fennema-Nio <[email protected]>
parent: Joe Conway <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Jelte Fennema-Nio @ 2024-08-19 11:57 UTC (permalink / raw)
To: Joe Conway <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On Mon, 19 Aug 2024 at 13:39, Joe Conway <[email protected]> wrote:
> What specifically? I am not seeing anything at the moment.
It seems to be working again fine. But before
https://git.postgresql.org was throwing 502 and 503 errors. Depending
on the request either by Varnish or nginx.
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 12:04 Robert Haas <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Robert Haas @ 2024-08-19 12:04 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; +Cc: Joe Conway <[email protected]>; Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On Mon, Aug 19, 2024 at 7:57 AM Jelte Fennema-Nio <[email protected]> wrote:
> On Mon, 19 Aug 2024 at 13:39, Joe Conway <[email protected]> wrote:
> > What specifically? I am not seeing anything at the moment.
>
> It seems to be working again fine. But before
> https://git.postgresql.org was throwing 502 and 503 errors. Depending
> on the request either by Varnish or nginx.
I'm still unable to access any https://git.postgresql.org/ URL.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 12:05 Jelte Fennema-Nio <[email protected]>
parent: Robert Haas <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Jelte Fennema-Nio @ 2024-08-19 12:05 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Joe Conway <[email protected]>; Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On Mon, 19 Aug 2024 at 14:04, Robert Haas <[email protected]> wrote:
> I'm still unable to access any https://git.postgresql.org/ URL.
Yeah it's broken for me again too.
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 12:06 Jelte Fennema-Nio <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Jelte Fennema-Nio @ 2024-08-19 12:06 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Joe Conway <[email protected]>; Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On Mon, 19 Aug 2024 at 14:05, Jelte Fennema-Nio <[email protected]> wrote:
>
> On Mon, 19 Aug 2024 at 14:04, Robert Haas <[email protected]> wrote:
> > I'm still unable to access any https://git.postgresql.org/ URL.
>
> Yeah it's broken for me again too.
In case the actual error is helpful (although I doubt it):
Error 503 Backend fetch failed
Backend fetch failed
Guru Meditation:
XID: 1030914118
________________________________
Varnish cache server
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 12:11 Joe Conway <[email protected]>
parent: Jelte Fennema-Nio <[email protected]>
0 siblings, 1 reply; 10+ messages in thread
From: Joe Conway @ 2024-08-19 12:11 UTC (permalink / raw)
To: Jelte Fennema-Nio <[email protected]>; Robert Haas <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; pgsql-hackers
On 8/19/24 08:06, Jelte Fennema-Nio wrote:
> On Mon, 19 Aug 2024 at 14:05, Jelte Fennema-Nio <[email protected]> wrote:
>>
>> On Mon, 19 Aug 2024 at 14:04, Robert Haas <[email protected]> wrote:
>> > I'm still unable to access any https://git.postgresql.org/ URL.
>>
>> Yeah it's broken for me again too.
>
> In case the actual error is helpful (although I doubt it):
>
> Error 503 Backend fetch failed
>
> Backend fetch failed
>
> Guru Meditation:
>
> XID: 1030914118
I am not seeing any errors here. nagios monitoring is showing an ipv6
error though. Not the same issue as yesterday.
--
Joe Conway
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 10+ messages in thread
* Re: gitmaster server problem?
@ 2024-08-19 14:24 Bruce Momjian <[email protected]>
parent: Joe Conway <[email protected]>
0 siblings, 0 replies; 10+ messages in thread
From: Bruce Momjian @ 2024-08-19 14:24 UTC (permalink / raw)
To: Joe Conway <[email protected]>; +Cc: Jelte Fennema-Nio <[email protected]>; Robert Haas <[email protected]>; Matthias van de Meent <[email protected]>; pgsql-hackers
On Mon, Aug 19, 2024 at 08:11:36AM -0400, Joe Conway wrote:
> On 8/19/24 08:06, Jelte Fennema-Nio wrote:
> > On Mon, 19 Aug 2024 at 14:05, Jelte Fennema-Nio <[email protected]> wrote:
> > >
> > > On Mon, 19 Aug 2024 at 14:04, Robert Haas <[email protected]> wrote:
> > > > I'm still unable to access any https://git.postgresql.org/ URL.
> > >
> > > Yeah it's broken for me again too.
> >
> > In case the actual error is helpful (although I doubt it):
> >
> > Error 503 Backend fetch failed
> >
> > Backend fetch failed
> >
> > Guru Meditation:
> >
> > XID: 1030914118
>
> I am not seeing any errors here. nagios monitoring is showing an ipv6 error
> though. Not the same issue as yesterday.
gitmaster.postgresql.org is working fine for me right now.
--
Bruce Momjian <[email protected]> https://momjian.us
EDB https://enterprisedb.com
Only you can decide what is important to you.
^ permalink raw reply [nested|flat] 10+ messages in thread
end of thread, other threads:[~2024-08-19 14:24 UTC | newest]
Thread overview: 10+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-03-10 08:35 [PATCH 6/8] Add default_toast_compression GUC Dilip Kumar <[email protected]>
2024-08-17 20:05 Re: gitmaster server problem? Joe Conway <[email protected]>
2024-08-19 09:04 ` Re: gitmaster server problem? Jelte Fennema-Nio <[email protected]>
2024-08-19 11:39 ` Re: gitmaster server problem? Joe Conway <[email protected]>
2024-08-19 11:57 ` Re: gitmaster server problem? Jelte Fennema-Nio <[email protected]>
2024-08-19 12:04 ` Re: gitmaster server problem? Robert Haas <[email protected]>
2024-08-19 12:05 ` Re: gitmaster server problem? Jelte Fennema-Nio <[email protected]>
2024-08-19 12:06 ` Re: gitmaster server problem? Jelte Fennema-Nio <[email protected]>
2024-08-19 12:11 ` Re: gitmaster server problem? Joe Conway <[email protected]>
2024-08-19 14:24 ` Re: gitmaster server problem? Bruce Momjian <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox