agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v3] Include the leader PID in logfile 29+ messages / 4 participants [nested] [flat]
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v3] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which adds the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 30 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index e806b13754..c525a0fd15 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6687,6 +6687,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>For a parallel worker, this is the Process ID of its leader + process. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7019,7 +7026,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7049,6 +7056,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index e4b717c79a..3055b72c01 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2448,6 +2449,25 @@ log_line_prefix(StringInfo buf, ErrorData *edata) else appendStringInfo(buf, "%d", MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader == NULL || leader->pid == MyProcPid) + /* padding only */ + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + else if (padding != 0) + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfo(buf, "%d", leader->pid); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': if (padding != 0) appendStringInfo(buf, "%*ld", padding, log_line_number); @@ -2836,6 +2856,16 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index aa30291ea3..55c2e23c47 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -536,6 +536,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --eJnRUKwClWJh1Khz-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v1 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which adds the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 8 +++- src/backend/utils/error/elog.c | 47 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index fc0e2c00c3..d53b62e2df 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6487,6 +6487,11 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%k</literal></entry> + <entry>Leader PID for a parallel process, NULL otherwise</entry> + <entry>yes</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -6777,7 +6782,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -6807,6 +6812,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 62eef7b71f..e6b3caf414 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -177,6 +178,7 @@ static void write_console(const char *line, int len); static void setup_formatted_log_time(void); static void setup_formatted_start_time(void); static const char *process_log_prefix_padding(const char *p, int *padding); +static pid_t get_leader_pid(); static void log_line_prefix(StringInfo buf, ErrorData *edata); static void write_csvlog(ErrorData *edata); static void send_message_to_server_log(ErrorData *edata); @@ -2402,6 +2404,24 @@ process_log_prefix_padding(const char *p, int *ppadding) return p; } +/* Return PID of leader, or InvalidPid if not a parallel worker */ +static pid_t +get_leader_pid() +{ + PGPROC *proc; + pid_t leader_pid = InvalidPid; + + if (MyBackendType != B_BG_WORKER) + return InvalidPid; + + LWLockAcquire(ProcArrayLock, LW_SHARED); + proc = BackendPidGetProcWithLock(MyProcPid); + if (proc && proc->lockGroupLeader) + leader_pid = proc->lockGroupLeader->pid; + LWLockRelease(ProcArrayLock); + return leader_pid; +} + /* * Format tag info for log lines; append to the provided buffer. */ @@ -2413,6 +2433,10 @@ log_line_prefix(StringInfo buf, ErrorData *edata) /* has counter been reset in current process? */ static int log_my_pid = 0; + + /* Leader PID is retrieved only once per process after forking from postmaster */ + static pid_t leader_pid = InvalidPid; + int padding; const char *p; @@ -2427,6 +2451,7 @@ log_line_prefix(StringInfo buf, ErrorData *edata) log_line_number = 0; log_my_pid = MyProcPid; formatted_start_time[0] = '\0'; + leader_pid = get_leader_pid(); } log_line_number++; @@ -2560,6 +2585,18 @@ log_line_prefix(StringInfo buf, ErrorData *edata) else appendStringInfo(buf, "%d", MyProcPid); break; + + case 'k': + if (!MyProcPort) + ; /* Do nothing */ + else if (leader_pid == InvalidPid) + ; /* Do nothing */ + else if (padding != 0) + appendStringInfo(buf, "%*d", padding, leader_pid); + else + appendStringInfo(buf, "%d", leader_pid); + break; + case 'l': if (padding != 0) appendStringInfo(buf, "%*ld", padding, log_line_number); @@ -2768,6 +2805,9 @@ write_csvlog(ErrorData *edata) /* has counter been reset in current process? */ static int log_my_pid = 0; + /* Leader PID is retrieved only once per process after forking from postmaster */ + static pid_t leader_pid = InvalidPid; + /* * This is one of the few places where we'd rather not inherit a static * variable's value from the postmaster. But since we will, reset it when @@ -2778,6 +2818,7 @@ write_csvlog(ErrorData *edata) log_line_number = 0; log_my_pid = MyProcPid; formatted_start_time[0] = '\0'; + leader_pid = get_leader_pid(); } log_line_number++; @@ -2948,6 +2989,12 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProcPort && leader_pid != InvalidPid) + appendStringInfo(&buf, "%d", leader_pid); + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index c0e9531f9c..32fc4b18e6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -530,6 +530,7 @@ # %r = remote host and port # %h = remote host # %p = process ID + # %k = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --QTprm0S8XgL7H0Dt-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which adds the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 11 ++++++- src/backend/utils/error/elog.c | 30 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index b353c61683..e60403d6a5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6611,6 +6611,14 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader if this process was + at some point involved in parallel query, otherwise null. For a + parallel group leader itself, this field is set to its own process + ID.</entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -6943,7 +6951,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -6973,6 +6981,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index e4b717c79a..dea098faf1 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2448,6 +2449,25 @@ log_line_prefix(StringInfo buf, ErrorData *edata) else appendStringInfo(buf, "%d", MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader == NULL) + /* padding only */ + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + else if (padding != 0) + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfo(buf, "%d", leader->pid); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': if (padding != 0) appendStringInfo(buf, "%*ld", padding, log_line_number); @@ -2836,6 +2856,16 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index e430e33c7b..48a50fb394 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -533,6 +533,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --cWoXeonUoKmBZSoM-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which adds the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 11 +++++++++- src/backend/utils/error/elog.c | 20 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 3cac340f32..f6ded2bc45 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6491,6 +6491,14 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%k</literal></entry> + <entry>Process ID of the parallel group leader if this + process is or was involved in parallel query, null + otherwise. For a parallel group leader, this field is + set to its own process ID.</entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -6789,7 +6797,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -6819,6 +6827,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 62eef7b71f..0f5881355d 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2560,6 +2561,18 @@ log_line_prefix(StringInfo buf, ErrorData *edata) else appendStringInfo(buf, "%d", MyProcPid); break; + + case 'k': + if (MyBackendType != B_BG_WORKER) + ; /* Do nothing */ + else if (!MyProc->lockGroupLeader) + ; /* Do nothing */ + else if (padding != 0) + appendStringInfo(buf, "%*d", padding, MyProc->lockGroupLeader->pid); + else + appendStringInfo(buf, "%d", MyProc->lockGroupLeader->pid); + break; + case 'l': if (padding != 0) appendStringInfo(buf, "%*ld", padding, log_line_number); @@ -2948,6 +2961,13 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyBackendType == B_BG_WORKER && + MyProc->lockGroupLeader) + appendStringInfo(&buf, "%d", MyProc->lockGroupLeader->pid); + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index aa44f0c9bf..6874792a15 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -530,6 +530,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %k = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --ikeVEW9yuYc//A+q-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v4] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 30 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..89dd19035c 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>For a parallel worker, this is the Process ID of its leader + process. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index e4b717c79a..3055b72c01 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2448,6 +2449,25 @@ log_line_prefix(StringInfo buf, ErrorData *edata) else appendStringInfo(buf, "%d", MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader == NULL || leader->pid == MyProcPid) + /* padding only */ + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + else if (padding != 0) + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfo(buf, "%d", leader->pid); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': if (padding != 0) appendStringInfo(buf, "%*ld", padding, log_line_number); @@ -2836,6 +2856,16 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --47eKBCiAZYFK5l32-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v3] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which adds the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 11 +++++++- src/backend/utils/error/elog.c | 27 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 672bf6f1ee..b9a1bd9a31 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6511,6 +6511,14 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%k</literal></entry> + <entry>Process ID of the parallel group leader if this + process is or was involved in parallel query, null + otherwise. For a parallel group leader, this field is + set to its own process ID.</entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -6809,7 +6817,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -6839,6 +6847,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index 62eef7b71f..5328664f4d 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2560,6 +2561,22 @@ log_line_prefix(StringInfo buf, ErrorData *edata) else appendStringInfo(buf, "%d", MyProcPid); break; + + case 'k': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader == NULL) + /* padding only */ + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + else if (padding != 0) + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfo(buf, "%d", leader->pid); + } + break; + case 'l': if (padding != 0) appendStringInfo(buf, "%*ld", padding, log_line_number); @@ -2948,6 +2965,16 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index aa44f0c9bf..6874792a15 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -530,6 +530,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %k = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --MdJ3p2aQbVp3YFXz-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* [PATCH v5 2/2] Include the leader PID in logfile @ 2020-03-14 03:03 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: Justin Pryzby @ 2020-03-14 03:03 UTC (permalink / raw) See also: b025f32e0b, which added the leader PID to pg_stat_activity --- doc/src/sgml/config.sgml | 10 ++++++- src/backend/utils/error/elog.c | 29 +++++++++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 994155ca00..e1da2e12e8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -6694,6 +6694,13 @@ local0.* /var/log/postgresql <entry>Process ID</entry> <entry>no</entry> </row> + <row> + <entry><literal>%P</literal></entry> + <entry>Process ID of the parallel group leader, if this process is + a parallel query worker. + </entry> + <entry>no</entry> + </row> <row> <entry><literal>%t</literal></entry> <entry>Time stamp without milliseconds</entry> @@ -7026,7 +7033,7 @@ log_line_prefix = '%m [%p] %q%u@%d/%a ' character count of the error position therein, location of the error in the PostgreSQL source code (if <varname>log_error_verbosity</varname> is set to <literal>verbose</literal>), - application name, and backend type. + application name, backend type, and leader PID. Here is a sample table definition for storing CSV-format log output: <programlisting> @@ -7056,6 +7063,7 @@ CREATE TABLE postgres_log location text, application_name text, backend_type text, + leader_pid integer, PRIMARY KEY (session_id, session_line_num) ); </programlisting> diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index b0b1a09ab1..60e2f524a7 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -77,6 +77,7 @@ #include "postmaster/syslogger.h" #include "storage/ipc.h" #include "storage/proc.h" +#include "storage/procarray.h" #include "tcop/tcopprot.h" #include "utils/guc.h" #include "utils/memutils.h" @@ -2425,6 +2426,23 @@ log_line_prefix(StringInfo buf, ErrorData *edata) case 'p': appendStringInfo(buf, "%*d", padding, MyProcPid); break; + + case 'P': + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + if (leader != NULL && leader->pid != MyProcPid) + /* leader_pid is blank for the leader itself */ + appendStringInfo(buf, "%*d", padding, leader->pid); + else + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + } + else if (padding != 0) + appendStringInfoSpaces(buf, + padding > 0 ? padding : -padding); + break; + case 'l': appendStringInfo(buf, "%*ld", padding, log_line_number); break; @@ -2766,6 +2784,17 @@ write_csvlog(ErrorData *edata) else appendCSVLiteral(&buf, GetBackendTypeDesc(MyBackendType)); + appendStringInfoChar(&buf, ','); + + /* leader PID */ + if (MyProc) + { + PGPROC *leader = MyProc->lockGroupLeader; + /* leader_pid is blank for leader itself */ + if (leader && leader->pid != MyProcPid) + appendStringInfo(&buf, "%d", leader->pid); + } + appendStringInfoChar(&buf, '\n'); /* If in the syslogger process, try to write messages direct to file */ diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b0715ae188..1ecdfc51a6 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -537,6 +537,7 @@ # %h = remote host # %b = backend type # %p = process ID + # %P = leader PID # %t = timestamp without milliseconds # %m = timestamp with milliseconds # %n = timestamp with milliseconds (as a Unix epoch) -- 2.17.0 --fDERRRNgB4on1jOB-- ^ permalink raw reply [nested|flat] 29+ messages in thread
* Re: Moving forward with TDE [PATCH v3] @ 2023-11-06 10:26 Matthias van de Meent <[email protected]> 2023-11-08 00:47 ` Re: Moving forward with TDE [PATCH v3] Andres Freund <[email protected]> 0 siblings, 1 reply; 29+ messages in thread From: Matthias van de Meent @ 2023-11-06 10:26 UTC (permalink / raw) To: Andres Freund <[email protected]>; +Cc: David Christensen <[email protected]>; vignesh C <[email protected]>; Aleksander Alekseev <[email protected]>; pgsql-hackers; Stephen Frost <[email protected]> On Sat, 4 Nov 2023 at 03:38, Andres Freund <[email protected]> wrote: > > Hi, > > On 2023-11-02 22:09:40 +0100, Matthias van de Meent wrote: > > I'm quite surprised at the significant number of changes being made > > outside the core storage manager files. I thought that changing out > > mdsmgr with an encrypted smgr (that could wrap mdsmgr if so desired) > > would be the most obvious change to implement cluster-wide encryption > > with the least code touched, as relations don't need to know whether > > the files they're writing are encrypted, right? Is there a reason to > > not implement this at the smgr level that I overlooked in the > > documentation of these patches? > > You can't really implement encryption transparently inside an smgr without > significant downsides. You need a way to store an initialization vector > associated with the page (or you can store that elsewhere, but then you've > doubled the worst cse amount of random reads/writes). The patch uses the LSN > as the IV (which I doubt is a good idea). For authenticated encryption further > additional storage space is required. I am unaware of any user of the smgr API that doesn't also use the buffer cache, and thus implicitly the Page layout with PageHeader [^1]. The API of smgr is also tailored to page-sized quanta of data with mostly relation-level information. I don't see why there would be a veil covering the layout of Page for smgr when all other information already points to the use of PageHeader and Page layouts. In my view, it would even make sense to allow the smgr to get exclusive access to some part of the page in the current Page layout. Yes, I agree that there will be an impact on usable page size if you want authenticated encryption, and that AMs will indeed need to account for storage space now being used by the smgr - inconvenient, but it serves a purpose. That would happen regardless of whether smgr or some higher system decides where to store the data for encryption - as long as it is on the page, the AM effectively can't use those bytes. But I'd say that's best solved by making the Page documentation and PageInit API explicit about the potential use of that space by the chosen storage method (encrypted, plain, ...) instead of requiring the various AMs to manually consider encryption when using Postgres' APIs for writing data to disk without hitting shared buffers; page space management is already a task of AMs, but handling the actual encryption is not. Should the AM really care whether the data on disk is encrypted or not? I don't think so. When the disk contains encrypted bytes, but smgrread() and smgrwrite() both produce and accept plaintext data, who's going to complain? Requiring AMs to be mindful about encryption on all common paths only adds pitfalls where encryption would be forgotten by the developer of AMs in one path or another. > To be able to to use the LSN as the IV, the patch needs to ensure that the LSN > increases in additional situations (a more aggressive version of > wal_log_hint_bits) - which can't be done below smgr, where we don't know about > what WAL logging was done. Nor can you easily just add space on the page below > md.c, for the purpose of storing an LSN independent IV and the authentication > data. I think that getting PageInit to allocate the smgr-specific area would take some effort, too (which would potentially require adding some relational context to PageInit, so that it knows which page of which relation it is going to initialize), but IMHO that would be more natural than requiring all index and table AMs to be aware the actual encryption of its pages and require manual handling of that encryption when the page needs to be written to disk, when it otherwise already conforms to the various buffer management and file extension APIs currently in use in PostgreSQL. I would expect "transparent" data encryption to be handled at the file write layer (i.e. smgr), not inside the AMs. Kind regards, Matthias van de Meent [^1] ReadBuffer_common uses PageIsVerifiedExtended which verifies that a page conforms with Postgres' Page layout if checksums are enabled. Furthermore, all builtin index AMs utilize pd_special, further implying the use of a PageInit/PageHeader-based page layout. Additionally, the heap tableAM also complies, and both FSM and VM also use postgres' Page layout. As for other AMs that I could check: bloom, rum, and pgvector's ivfflat and hnsw all use page layouts. ^ permalink raw reply [nested|flat] 29+ messages in thread
* Re: Moving forward with TDE [PATCH v3] 2023-11-06 10:26 Re: Moving forward with TDE [PATCH v3] Matthias van de Meent <[email protected]> @ 2023-11-08 00:47 ` Andres Freund <[email protected]> 2023-11-08 22:47 ` Re: Moving forward with TDE [PATCH v3] David Christensen <[email protected]> 0 siblings, 1 reply; 29+ messages in thread From: Andres Freund @ 2023-11-08 00:47 UTC (permalink / raw) To: Matthias van de Meent <[email protected]>; +Cc: David Christensen <[email protected]>; vignesh C <[email protected]>; Aleksander Alekseev <[email protected]>; pgsql-hackers; Stephen Frost <[email protected]> Hi, On 2023-11-06 11:26:44 +0100, Matthias van de Meent wrote: > On Sat, 4 Nov 2023 at 03:38, Andres Freund <[email protected]> wrote: > > On 2023-11-02 22:09:40 +0100, Matthias van de Meent wrote: > > > I'm quite surprised at the significant number of changes being made > > > outside the core storage manager files. I thought that changing out > > > mdsmgr with an encrypted smgr (that could wrap mdsmgr if so desired) > > > would be the most obvious change to implement cluster-wide encryption > > > with the least code touched, as relations don't need to know whether > > > the files they're writing are encrypted, right? Is there a reason to > > > not implement this at the smgr level that I overlooked in the > > > documentation of these patches? > > > > You can't really implement encryption transparently inside an smgr without > > significant downsides. You need a way to store an initialization vector > > associated with the page (or you can store that elsewhere, but then you've > > doubled the worst cse amount of random reads/writes). The patch uses the LSN > > as the IV (which I doubt is a good idea). For authenticated encryption further > > additional storage space is required. > > I am unaware of any user of the smgr API that doesn't also use the > buffer cache, and thus implicitly the Page layout with PageHeader > [^1] Everything indeed uses a PageHeader - but there are a number of places that do *not* utilize pd_lower/upper/special. E.g. visibilitymap.c just assumes that those fields are zero - and changing that wouldn't be trivial / free, because we do a lot of bitmasking/shifting with constants derived from #define MAPSIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) which obviously wouldn't be constant anymore if you could reserve space on the page. > The API of smgr is also tailored to page-sized quanta of data > with mostly relation-level information. I don't see why there would be > a veil covering the layout of Page for smgr when all other information > already points to the use of PageHeader and Page layouts. In my view, > it would even make sense to allow the smgr to get exclusive access to > some part of the page in the current Page layout. > > Yes, I agree that there will be an impact on usable page size if you > want authenticated encryption, and that AMs will indeed need to > account for storage space now being used by the smgr - inconvenient, > but it serves a purpose. That would happen regardless of whether smgr > or some higher system decides where to store the data for encryption - > as long as it is on the page, the AM effectively can't use those > bytes. > But I'd say that's best solved by making the Page documentation and > PageInit API explicit about the potential use of that space by the > chosen storage method (encrypted, plain, ...) instead of requiring the > various AMs to manually consider encryption when using Postgres' APIs > for writing data to disk without hitting shared buffers; page space > management is already a task of AMs, but handling the actual > encryption is not. I don't particularly disagree with any detail here - but to me reserving space for nonces etc at PageInit() time pretty much is the opposite of handling encryption inside smgr. > Should the AM really care whether the data on disk is encrypted or > not? I don't think so. When the disk contains encrypted bytes, but > smgrread() and smgrwrite() both produce and accept plaintext data, > who's going to complain? Requiring AMs to be mindful about encryption > on all common paths only adds pitfalls where encryption would be > forgotten by the developer of AMs in one path or another. I agree with that - I think the way the patch currently is designed is not right. There's other stuff you can't trivially do at the smgr level. E.g. if checksums or encryption is enabled, you need to copy the buffer to compute checksums / do IO if in shared buffers, because somebody could set a hint bit even with just a shared content lock. But you don't need that when coming from private buffers during index builds. > I think that getting PageInit to allocate the smgr-specific area would > take some effort, too (which would potentially require adding some > relational context to PageInit, so that it knows which page of which > relation it is going to initialize), but IMHO that would be more > natural than requiring all index and table AMs to be aware the actual > encryption of its pages and require manual handling of that encryption > when the page needs to be written to disk, when it otherwise already > conforms to the various buffer management and file extension APIs > currently in use in PostgreSQL. I would expect "transparent" data > encryption to be handled at the file write layer (i.e. smgr), not > inside the AMs. As mentioned above - I agree that the relevant code shouldn't be in index AMs. But I somewhat doubt that smgr is the right level either. For one, the place computing checksums needs awareness of locking / sharing semantics as well as knowledge about WAL logging. That's IMO above smgr. For another, if we ever got another smgr implementation - should it have to reimplement encryption? ISTM that there's a layer missing. Places bypassing bufmgr.c currently need their own handling of checksums (and in the future encryption), because the relevant bufmgr.c code can't be reached without pages in the buffer pool. Which is why we have PageIsVerifiedExtended() and PageSetChecksumInplace() calls in gist, hash, heapam, nbtree ... IMO when checksums were added, we should have added the proper abstraction layer instead of littering the code with redundant copies. Greetings, Andres Freund ^ permalink raw reply [nested|flat] 29+ messages in thread
* Re: Moving forward with TDE [PATCH v3] 2023-11-06 10:26 Re: Moving forward with TDE [PATCH v3] Matthias van de Meent <[email protected]> 2023-11-08 00:47 ` Re: Moving forward with TDE [PATCH v3] Andres Freund <[email protected]> @ 2023-11-08 22:47 ` David Christensen <[email protected]> 0 siblings, 0 replies; 29+ messages in thread From: David Christensen @ 2023-11-08 22:47 UTC (permalink / raw) To: Andres Freund <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; vignesh C <[email protected]>; Aleksander Alekseev <[email protected]>; pgsql-hackers; Stephen Frost <[email protected]> On Tue, Nov 7, 2023 at 6:47 PM Andres Freund <[email protected]> wrote: > Hi, > > On 2023-11-06 11:26:44 +0100, Matthias van de Meent wrote: > > On Sat, 4 Nov 2023 at 03:38, Andres Freund <[email protected]> wrote: > > > On 2023-11-02 22:09:40 +0100, Matthias van de Meent wrote: > > > > I'm quite surprised at the significant number of changes being made > > > > outside the core storage manager files. I thought that changing out > > > > mdsmgr with an encrypted smgr (that could wrap mdsmgr if so desired) > > > > would be the most obvious change to implement cluster-wide encryption > > > > with the least code touched, as relations don't need to know whether > > > > the files they're writing are encrypted, right? Is there a reason to > > > > not implement this at the smgr level that I overlooked in the > > > > documentation of these patches? > > > > > > You can't really implement encryption transparently inside an smgr > without > > > significant downsides. You need a way to store an initialization vector > > > associated with the page (or you can store that elsewhere, but then > you've > > > doubled the worst cse amount of random reads/writes). The patch uses > the LSN > > > as the IV (which I doubt is a good idea). For authenticated encryption > further > > > additional storage space is required. > > > > I am unaware of any user of the smgr API that doesn't also use the > > buffer cache, and thus implicitly the Page layout with PageHeader > > [^1] > > Everything indeed uses a PageHeader - but there are a number of places > that do > *not* utilize pd_lower/upper/special. E.g. visibilitymap.c just assumes > that > those fields are zero - and changing that wouldn't be trivial / free, > because > we do a lot of bitmasking/shifting with constants derived from > > #define MAPSIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) > > which obviously wouldn't be constant anymore if you could reserve space on > the > page. > While not constants, I was able to get this working with variable values here in a way that did not have the overhead of the original patch for the vismap specifically using Montgomery Multiplication for division/mod. This was actually the heaviest of the changes from moving to runtime-calculated, so we might be able to use this approach in this specific case even if only this change is required for this specific fork. > > The API of smgr is also tailored to page-sized quanta of data > > with mostly relation-level information. I don't see why there would be > > a veil covering the layout of Page for smgr when all other information > > already points to the use of PageHeader and Page layouts. In my view, > > it would even make sense to allow the smgr to get exclusive access to > > some part of the page in the current Page layout. > > > > Yes, I agree that there will be an impact on usable page size if you > > want authenticated encryption, and that AMs will indeed need to > > account for storage space now being used by the smgr - inconvenient, > > but it serves a purpose. That would happen regardless of whether smgr > > or some higher system decides where to store the data for encryption - > > as long as it is on the page, the AM effectively can't use those > > bytes. > > But I'd say that's best solved by making the Page documentation and > > PageInit API explicit about the potential use of that space by the > > chosen storage method (encrypted, plain, ...) instead of requiring the > > various AMs to manually consider encryption when using Postgres' APIs > > for writing data to disk without hitting shared buffers; page space > > management is already a task of AMs, but handling the actual > > encryption is not. > > I don't particularly disagree with any detail here - but to me reserving > space > for nonces etc at PageInit() time pretty much is the opposite of handling > encryption inside smgr. > Originally, I was anticipating that we might want different space amounts reserved on different classes of pages (apart from encryption), so while we'd be storing the default page reserved size in pg_control we'd not be limited to this in the structure of the page calls. We could presumably just move the logic into PageInit() itself if every reserved allocation is the same and individual call sites wouldn't need to know about it. The call sites do have more context as to the requirements of the page or the "type" of page in play, which if we made it dependent on page type would need to get passed in somehow, which was where the reserved_page_size parameter came in to the current patch. > > > Should the AM really care whether the data on disk is encrypted or > > not? I don't think so. When the disk contains encrypted bytes, but > > smgrread() and smgrwrite() both produce and accept plaintext data, > > who's going to complain? Requiring AMs to be mindful about encryption > > on all common paths only adds pitfalls where encryption would be > > forgotten by the developer of AMs in one path or another. > > I agree with that - I think the way the patch currently is designed is not > right. > The things that need to care tend to be the same places that need to care about setting checksums, that or being aware of the LSNs in play or needing to be set. I'd agree that a common interface for "get this page ready for writing to storage" and "get this page converted from storage" which could handle both checksums, encryption, or additional page features would make sense. (I doubt we'd want hooks to support page in/page out, but if we /did/ want that, this'd also likely live there.) There's other stuff you can't trivially do at the smgr level. E.g. if > checksums or encryption is enabled, you need to copy the buffer to compute > checksums / do IO if in shared buffers, because somebody could set a hint > bit > even with just a shared content lock. But you don't need that when coming > from > private buffers during index builds. > > > > > I think that getting PageInit to allocate the smgr-specific area would > > take some effort, too (which would potentially require adding some > > relational context to PageInit, so that it knows which page of which > > relation it is going to initialize), but IMHO that would be more > > natural than requiring all index and table AMs to be aware the actual > > encryption of its pages and require manual handling of that encryption > > when the page needs to be written to disk, when it otherwise already > > conforms to the various buffer management and file extension APIs > > currently in use in PostgreSQL. I would expect "transparent" data > > encryption to be handled at the file write layer (i.e. smgr), not > > inside the AMs. > > As mentioned above - I agree that the relevant code shouldn't be in index > AMs. But I somewhat doubt that smgr is the right level either. For one, the > place computing checksums needs awareness of locking / sharing semantics as > well as knowledge about WAL logging. That's IMO above smgr. For another, > if we > ever got another smgr implementation - should it have to reimplement > encryption? > > ISTM that there's a layer missing. Places bypassing bufmgr.c currently need > their own handling of checksums (and in the future encryption), because the > relevant bufmgr.c code can't be reached without pages in the buffer > pool. Which is why we have PageIsVerifiedExtended() and > PageSetChecksumInplace() calls in gist, hash, heapam, nbtree ... IMO when > checksums were added, we should have added the proper abstraction layer > instead of littering the code with redundant copies. Agreed that a fixup patch to add /something/ here would be good. A concern here is what context would need to be passed in; certainly with only checksums just a Page is sufficient, but if we have AAD we'd need to be able to pass that in or otherwise be able to identify it in the page. As a point of references, the existing GCM patch authenticates all unencrypted header fields (i.e., PageHeaderData up to the pd_special field), plus the RelFileNumber and BlockNumber, of which we'd need to pass in RelFileNumber and BlockNumber (and presumably would want the ForkNum as well if expanding the set of authenticated data). To some extent, we can punt on some of this, as the existing call sites have been modified in this patch to pass that info in already, so it's really about how we marshal that data. Thanks, David ^ permalink raw reply [nested|flat] 29+ messages in thread
end of thread, other threads:[~2023-11-08 22:47 UTC | newest] Thread overview: 29+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v3] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v1 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v4] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v3] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2020-03-14 03:03 [PATCH v5 2/2] Include the leader PID in logfile Justin Pryzby <[email protected]> 2023-11-06 10:26 Re: Moving forward with TDE [PATCH v3] Matthias van de Meent <[email protected]> 2023-11-08 00:47 ` Re: Moving forward with TDE [PATCH v3] Andres Freund <[email protected]> 2023-11-08 22:47 ` Re: Moving forward with TDE [PATCH v3] David Christensen <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox