public inbox for [email protected]
help / color / mirror / Atom feedFrom: Kirill Gavrilov <[email protected]>
To: Jim Jones <[email protected]>
Cc: Andrey M. Borodin <[email protected]>
Cc: Euler Taveira <[email protected]>
Cc: [email protected]
Subject: Re: Truncate logs by max_log_size
Date: Tue, 1 Oct 2024 18:46:32 +0300
Message-ID: <CA+E0NR4=J-pgqEAV2weymygGmk1ePym45m2AcGyi2hpa9xRvBg@mail.gmail.com> (raw)
In-Reply-To: <[email protected]>
References: <CA+E0NR4S+NC6+QHyY_vUuQZMzLhKqczMx-jJVqtjAxF6+=JwAA@mail.gmail.com>
<[email protected]>
<[email protected]>
<[email protected]>
On Tue, Oct 1, 2024 at 2:46 PM Jim Jones <[email protected]> wrote:
>
> On 27.09.24 12:36, Andrey M. Borodin wrote:
> > Consider max_log_size = 10Mb. The perspective might look very different.
> > It’s not about WHERE anymore. It's a guard against heavy abuse.
> >
> > The feature looks very important for me.
> I have the same opinion. As a fail safe it sounds very useful to me.
>
> Unfortunately, the patch does not apply:
>
> $ git apply
> ~/patches/max_log_query/0001-parameter-max_log_size-to-truncate-logs.patch
> -v
> Checking patch src/backend/utils/error/elog.c...
> error: while searching for:
> char *Log_destination_string = NULL;
> bool syslog_sequence_numbers = true;
> bool syslog_split_messages = true;
>
> /* Processed form of backtrace_symbols GUC */
> static char *backtrace_symbol_list;
>
> error: patch failed: src/backend/utils/error/elog.c:114
> error: src/backend/utils/error/elog.c: patch does not apply
> Checking patch src/backend/utils/misc/guc_tables.c...
> Hunk #1 succeeded at 3714 (offset 247 lines).
> Checking patch src/backend/utils/misc/postgresql.conf.sample...
> Hunk #1 succeeded at 615 (offset 23 lines).
> Checking patch src/bin/pg_ctl/t/004_logrotate.pl...
> error: while searching for:
> check_log_pattern('csvlog', $new_current_logfiles, 'syntax error', $node);
> check_log_pattern('jsonlog', $new_current_logfiles, 'syntax error', $node);
>
> $node->stop();
>
> done_testing();
>
> error: patch failed: src/bin/pg_ctl/t/004_logrotate.pl:135
> error: src/bin/pg_ctl/t/004_logrotate.pl: patch does not apply
> Checking patch src/include/utils/elog.h...
> Hunk #1 succeeded at 502 (offset 1 line).
>
> Does it truncate only single queries or also a transaction with many
> statements?
>
> Thanks!
>
> Best, Jim
>
My apologies, attached patch should work on master branch.
It truncates single queries and a statement in transaction that occurred an
error.
Attachments:
[application/octet-stream] 0002-parameter-max_log_size-to-truncate-logs.patch (4.7K, ../CA+E0NR4=J-pgqEAV2weymygGmk1ePym45m2AcGyi2hpa9xRvBg@mail.gmail.com/3-0002-parameter-max_log_size-to-truncate-logs.patch)
download | inline diff:
From d2eb9ccde970f752be12ce4dbcf40c55806599c9 Mon Sep 17 00:00:00 2001
From: diphantxm <[email protected]>
Date: Tue, 1 Oct 2024 17:28:16 +0300
Subject: [PATCH] parameter max_log_size to truncate logs
There is no need to log the entire query, because it may be large and take lots of space on disk. Parameter max_log_size set the maximum length for logged query. Everything beyond that length is truncated. Value 0 disables the parameter.
---
src/backend/utils/error/elog.c | 8 ++++++++
src/backend/utils/misc/guc_tables.c | 11 +++++++++++
src/backend/utils/misc/postgresql.conf.sample | 2 ++
src/bin/pg_ctl/t/004_logrotate.pl | 15 +++++++++++++++
src/include/utils/elog.h | 1 +
5 files changed, 37 insertions(+)
diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c
index 5cbb5b5416..0690ec57ce 100644
--- a/src/backend/utils/error/elog.c
+++ b/src/backend/utils/error/elog.c
@@ -111,6 +111,7 @@ int Log_destination = LOG_DESTINATION_STDERR;
char *Log_destination_string = NULL;
bool syslog_sequence_numbers = true;
bool syslog_split_messages = true;
+int max_log_size = 0;
/* Processed form of backtrace_functions GUC */
static char *backtrace_function_list;
@@ -1693,6 +1694,13 @@ EmitErrorReport(void)
CHECK_STACK_DEPTH();
oldcontext = MemoryContextSwitchTo(edata->assoc_context);
+ if (max_log_size != 0 && debug_query_string != NULL)
+ {
+ char* str = debug_query_string;
+ str[pg_mbcliplen(str, strlen(str), max_log_size)] = '\0';
+ debug_query_string = str;
+ }
+
/*
* Reset the formatted timestamp fields before emitting any logs. This
* includes all the log destinations and emit_log_hook, as the latter
diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c
index 686309db58..cf1404c059 100644
--- a/src/backend/utils/misc/guc_tables.c
+++ b/src/backend/utils/misc/guc_tables.c
@@ -3714,6 +3714,17 @@ struct config_int ConfigureNamesInt[] =
NULL, NULL, NULL
},
+ {
+ {"max_log_size", PGC_SIGHUP, LOGGING_WHAT,
+ gettext_noop("Sets max size of logged statement."),
+ NULL
+ },
+ &max_log_size,
+ 5 * (1024 * 1024),
+ 0, INT_MAX,
+ NULL, NULL, NULL
+ },
+
/* End-of-list marker */
{
{NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 667e0dc40a..0ffb32d5fb 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -615,6 +615,8 @@
# bind-parameter values to N bytes;
# -1 means print in full, 0 disables
#log_statement = 'none' # none, ddl, mod, all
+#max_log_size = 0 # max size of logged statement_timeout
+ # 0 disables the feature
#log_replication_commands = off
#log_temp_files = -1 # log temporary files equal or larger
# than the specified size in kilobytes;
diff --git a/src/bin/pg_ctl/t/004_logrotate.pl b/src/bin/pg_ctl/t/004_logrotate.pl
index eacca1a652..1c1f89cb0a 100644
--- a/src/bin/pg_ctl/t/004_logrotate.pl
+++ b/src/bin/pg_ctl/t/004_logrotate.pl
@@ -69,6 +69,7 @@ log_destination = 'stderr, csvlog, jsonlog'
# these ensure stability of test results:
log_rotation_age = 0
lc_messages = 'C'
+max_log_size = 32
));
$node->start();
@@ -135,6 +136,20 @@ check_log_pattern('stderr', $new_current_logfiles, 'syntax error', $node);
check_log_pattern('csvlog', $new_current_logfiles, 'syntax error', $node);
check_log_pattern('jsonlog', $new_current_logfiles, 'syntax error', $node);
+$node->psql('postgres', 'INSERT INTO SOME_NON_EXISTANT_TABLE VALUES (TEST)');
+for (my $attempts = 0; $attempts < $max_attempts; $attempts++)
+{
+ eval {
+ $current_logfiles = slurp_file($node->data_dir . '/current_logfiles');
+ };
+ last unless $@;
+ usleep(100_000);
+}
+die $@ if $@;
+check_log_pattern('stderr', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node);
+check_log_pattern('csvlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node);
+check_log_pattern('jsonlog', $current_logfiles, 'INSERT INTO SOME_NON_EXISTANT_TA(?!(BLE VALUES \(TEST\)))', $node);
+
$node->stop();
done_testing();
diff --git a/src/include/utils/elog.h b/src/include/utils/elog.h
index e54eca5b48..a8e30006f2 100644
--- a/src/include/utils/elog.h
+++ b/src/include/utils/elog.h
@@ -502,6 +502,7 @@ extern PGDLLIMPORT int Log_destination;
extern PGDLLIMPORT char *Log_destination_string;
extern PGDLLIMPORT bool syslog_sequence_numbers;
extern PGDLLIMPORT bool syslog_split_messages;
+extern PGDLLIMPORT int max_log_size;
/* Log destination bitmap */
#define LOG_DESTINATION_STDERR 1
--
2.39.2 (Apple Git-143)
view thread (2+ messages)
reply
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Reply to all the recipients using the --to and --cc options:
reply via email
To: [email protected]
Cc: [email protected], [email protected], [email protected], [email protected], [email protected]
Subject: Re: Truncate logs by max_log_size
In-Reply-To: <CA+E0NR4=J-pgqEAV2weymygGmk1ePym45m2AcGyi2hpa9xRvBg@mail.gmail.com>
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox