public inbox for [email protected]
help / color / mirror / Atom feed[PATCH v10] Generated columns
85+ messages / 17 participants
[nested] [flat]
* [PATCH v10] Generated columns
@ 2019-03-26 13:31 Peter Eisentraut <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Peter Eisentraut @ 2019-03-26 13:31 UTC (permalink / raw)
This is an SQL-standard feature that allows creating columns that are
computed from expressions rather than assigned, similar to a view or
materialized view but on a column basis.
This implement one kind of generated column: stored (computed on
write). Another kind, virtual (computed on read), is planned for the
future, and some room is left for it.
Discussion: https://www.postgresql.org/message-id/flat/[email protected]
---
.../postgres_fdw/expected/postgres_fdw.out | 25 +
contrib/postgres_fdw/postgres_fdw.c | 3 +-
contrib/postgres_fdw/sql/postgres_fdw.sql | 14 +
doc/src/sgml/catalogs.sgml | 19 +-
doc/src/sgml/ddl.sgml | 118 +++
doc/src/sgml/information_schema.sgml | 66 +-
doc/src/sgml/protocol.sgml | 4 +-
doc/src/sgml/ref/copy.sgml | 3 +-
doc/src/sgml/ref/create_foreign_table.sgml | 27 +-
doc/src/sgml/ref/create_table.sgml | 45 +-
doc/src/sgml/ref/create_trigger.sgml | 4 +-
doc/src/sgml/textsearch.sgml | 26 +-
doc/src/sgml/trigger.sgml | 18 +
src/backend/access/common/tupdesc.c | 11 +
src/backend/catalog/heap.c | 95 ++-
src/backend/catalog/information_schema.sql | 30 +-
src/backend/commands/copy.c | 31 +-
src/backend/commands/tablecmds.c | 167 +++-
src/backend/commands/trigger.c | 31 +-
src/backend/commands/typecmds.c | 6 +-
src/backend/executor/execMain.c | 8 +-
src/backend/executor/execReplication.c | 11 +
src/backend/executor/nodeModifyTable.c | 112 +++
src/backend/nodes/copyfuncs.c | 2 +
src/backend/nodes/equalfuncs.c | 2 +
src/backend/nodes/outfuncs.c | 9 +
src/backend/nodes/readfuncs.c | 1 +
src/backend/optimizer/plan/createplan.c | 8 +-
src/backend/optimizer/util/inherit.c | 6 +
src/backend/optimizer/util/plancat.c | 19 +
src/backend/parser/analyze.c | 27 +
src/backend/parser/gram.y | 14 +-
src/backend/parser/parse_agg.c | 11 +
src/backend/parser/parse_expr.c | 5 +
src/backend/parser/parse_func.c | 3 +
src/backend/parser/parse_relation.c | 19 +
src/backend/parser/parse_utilcmd.c | 66 +-
src/backend/replication/logical/proto.c | 9 +-
src/backend/replication/logical/relation.c | 2 +-
src/backend/replication/logical/tablesync.c | 6 +-
src/backend/replication/logical/worker.c | 2 +-
src/backend/replication/pgoutput/pgoutput.c | 2 +-
src/backend/rewrite/rewriteHandler.c | 36 +-
src/backend/utils/cache/lsyscache.c | 33 +
src/backend/utils/cache/partcache.c | 1 +
src/backend/utils/cache/relcache.c | 4 +
src/bin/pg_dump/pg_dump.c | 40 +-
src/bin/pg_dump/pg_dump.h | 1 +
src/bin/pg_dump/pg_dump_sort.c | 10 +
src/bin/pg_dump/t/002_pg_dump.pl | 17 +
src/bin/psql/describe.c | 23 +-
src/include/access/tupdesc.h | 1 +
src/include/catalog/heap.h | 4 +-
src/include/catalog/pg_attribute.h | 5 +
src/include/catalog/pg_class.dat | 2 +-
src/include/executor/nodeModifyTable.h | 2 +
src/include/nodes/execnodes.h | 3 +
src/include/nodes/parsenodes.h | 24 +-
src/include/optimizer/plancat.h | 2 +
src/include/parser/kwlist.h | 1 +
src/include/parser/parse_node.h | 3 +-
src/include/utils/lsyscache.h | 1 +
src/pl/plperl/expected/plperl_trigger.out | 95 +++
src/pl/plperl/plperl.c | 40 +-
src/pl/plperl/sql/plperl_trigger.sql | 36 +
src/pl/plpgsql/src/pl_exec.c | 20 +
src/pl/plpython/expected/plpython_trigger.out | 94 +++
src/pl/plpython/plpy_cursorobject.c | 5 +-
src/pl/plpython/plpy_exec.c | 23 +-
src/pl/plpython/plpy_spi.c | 3 +-
src/pl/plpython/plpy_typeio.c | 17 +-
src/pl/plpython/plpy_typeio.h | 2 +-
src/pl/plpython/sql/plpython_trigger.sql | 37 +
src/pl/tcl/expected/pltcl_trigger.out | 99 +++
src/pl/tcl/pltcl.c | 50 +-
src/pl/tcl/sql/pltcl_trigger.sql | 36 +
.../regress/expected/create_table_like.out | 46 ++
src/test/regress/expected/generated.out | 768 ++++++++++++++++++
src/test/regress/parallel_schedule | 2 +-
src/test/regress/serial_schedule | 1 +
src/test/regress/sql/create_table_like.sql | 14 +
src/test/regress/sql/generated.sql | 451 ++++++++++
src/test/subscription/t/011_generated.pl | 65 ++
83 files changed, 3055 insertions(+), 149 deletions(-)
create mode 100644 src/test/regress/expected/generated.out
create mode 100644 src/test/regress/sql/generated.sql
create mode 100644 src/test/subscription/t/011_generated.pl
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 2be67bca02..8b3768028f 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -6431,6 +6431,31 @@ select * from rem1;
11 | bye remote
(4 rows)
+-- ===================================================================
+-- test generated columns
+-- ===================================================================
+create table gloc1 (a int, b int);
+alter table gloc1 set (autovacuum_enabled = 'false');
+create foreign table grem1 (
+ a int,
+ b int generated always as (a * 2) stored)
+ server loopback options(table_name 'gloc1');
+insert into grem1 (a) values (1), (2);
+update grem1 set a = 22 where a = 2;
+select * from gloc1;
+ a | b
+----+----
+ 1 | 2
+ 22 | 44
+(2 rows)
+
+select * from grem1;
+ a | b
+----+----
+ 1 | 2
+ 22 | 44
+(2 rows)
+
-- ===================================================================
-- test local triggers
-- ===================================================================
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 2f387fac42..d0d36aaa0d 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -1644,9 +1644,10 @@ postgresPlanForeignModify(PlannerInfo *root,
else if (operation == CMD_UPDATE)
{
int col;
+ Bitmapset *allUpdatedCols = bms_union(rte->updatedCols, rte->extraUpdatedCols);
col = -1;
- while ((col = bms_next_member(rte->updatedCols, col)) >= 0)
+ while ((col = bms_next_member(allUpdatedCols, col)) >= 0)
{
/* bit numbers are offset by FirstLowInvalidHeapAttributeNumber */
AttrNumber attno = col + FirstLowInvalidHeapAttributeNumber;
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 4728511abf..a96c327134 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -1363,6 +1363,20 @@ CREATE VIEW rw_view AS SELECT * FROM parent_tbl
select * from loc1;
select * from rem1;
+-- ===================================================================
+-- test generated columns
+-- ===================================================================
+create table gloc1 (a int, b int);
+alter table gloc1 set (autovacuum_enabled = 'false');
+create foreign table grem1 (
+ a int,
+ b int generated always as (a * 2) stored)
+ server loopback options(table_name 'gloc1');
+insert into grem1 (a) values (1), (2);
+update grem1 set a = 22 where a = 2;
+select * from gloc1;
+select * from grem1;
+
-- ===================================================================
-- test local triggers
-- ===================================================================
diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 45ed077654..bf3ab132ec 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -1129,9 +1129,11 @@ <title><structname>pg_attribute</structname> Columns</title>
<entry><type>bool</type></entry>
<entry></entry>
<entry>
- This column has a default value, in which case there will be a
- corresponding entry in the <structname>pg_attrdef</structname>
- catalog that actually defines the value.
+ This column has a default expression or generation expression, in which
+ case there will be a corresponding entry in the
+ <structname>pg_attrdef</structname> catalog that actually defines the
+ expression. (Check <structfield>attgenerated</structfield> to
+ determine whether this is a default or a generation expression.)
</entry>
</row>
@@ -1159,6 +1161,17 @@ <title><structname>pg_attribute</structname> Columns</title>
</entry>
</row>
+ <row>
+ <entry><structfield>attgenerated</structfield></entry>
+ <entry><type>char</type></entry>
+ <entry></entry>
+ <entry>
+ If a zero byte (<literal>''</literal>), then not a generated column.
+ Otherwise, <literal>s</literal> = stored. (Other values might be added
+ in the future.)
+ </entry>
+ </row>
+
<row>
<entry><structfield>attisdropped</structfield></entry>
<entry><type>bool</type></entry>
diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml
index 110f6b4657..1fe27c5da9 100644
--- a/doc/src/sgml/ddl.sgml
+++ b/doc/src/sgml/ddl.sgml
@@ -233,6 +233,124 @@ <title>Default Values</title>
</para>
</sect1>
+ <sect1 id="ddl-generated-columns">
+ <title>Generated Columns</title>
+
+ <indexterm zone="ddl-generated-columns">
+ <primary>generated column</primary>
+ </indexterm>
+
+ <para>
+ A generated column is a special column that is always computed from other
+ columns. Thus, it is for columns what a view is for tables. There are two
+ kinds of generated columns: stored and virtual. A stored generated column
+ is computed when it is written (inserted or updated) and occupies storage
+ as if it were a normal column. A virtual generated column occupies no
+ storage and is computed when it is read. Thus, a virtual generated column
+ is similar to a view and a stored generated column is similar to a
+ materialized view (except that it is always updated automatically).
+ PostgreSQL currently implements only stored generated columns.
+ </para>
+
+ <para>
+ To create a generated column, use the <literal>GENERATED ALWAYS
+ AS</literal> clause in <command>CREATE TABLE</command>, for example:
+<programlisting>
+CREATE TABLE people (
+ ...,
+ height_cm numeric,
+ height_in numeric <emphasis>GENERATED ALWAYS AS (height_cm * 2.54) STORED</emphasis>
+);
+</programlisting>
+ The keyword <literal>STORED</literal> must be specified to choose the
+ stored kind of generated column. See <xref linkend="sql-createtable"/> for
+ more details.
+ </para>
+
+ <para>
+ A generated column cannot be written to directly. In
+ <command>INSERT</command> or <command>UPDATE</command> commands, a value
+ cannot be specified for a generated column, but the keyword
+ <literal>DEFAULT</literal> may be specified.
+ </para>
+
+ <para>
+ Consider the differences between a column with a default and a generated
+ column. The column default is evaluated once when the row is first
+ inserted if no other value was provided; a generated column is updated
+ whenever the row changes and cannot be overridden. A column default may
+ not refer to other columns of the table; a generation expression would
+ normally do so. A column default can use volatile functions, for example
+ <literal>random()</literal> or functions referring to the current time;
+ this is not allowed for generated columns.
+ </para>
+
+ <para>
+ Several restrictions apply to the definition of generated columns and
+ tables involving generated columns:
+
+ <itemizedlist>
+ <listitem>
+ <para>
+ The generation expression can only use immutable functions and cannot
+ use subqueries or reference anything other than the current row in any
+ way.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A generation expression cannot reference another generated column.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A generation expression cannot reference a system column, except
+ <varname>tableoid</varname>.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A generated column cannot have a column default or an identity definition.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A generated column cannot be part of a partition key.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Foreign tables can have generated columns. See <xref
+ linkend="sql-createforeigntable"/> for details.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ Additional considerations apply to the use of generated columns.
+ <itemizedlist>
+ <listitem>
+ <para>
+ Generated columns maintain access privileges separately from their
+ underlying base columns. So, it is possible to arrange it so that a
+ particular role can read from a generated column but not from the
+ underlying base columns.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Generated columns are, conceptually, updated after
+ <literal>BEFORE</literal> triggers have run. Therefore, changes made to
+ base columns in a <literal>BEFORE</literal> trigger will be reflected in
+ generated columns. But conversely, it is not allowed to access
+ generated columns in <literal>BEFORE</literal> triggers.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+ </sect1>
+
<sect1 id="ddl-constraints">
<title>Constraints</title>
diff --git a/doc/src/sgml/information_schema.sgml b/doc/src/sgml/information_schema.sgml
index b13700da92..1321ade44a 100644
--- a/doc/src/sgml/information_schema.sgml
+++ b/doc/src/sgml/information_schema.sgml
@@ -952,6 +952,62 @@ <title><literal>collation_character_set_applicability</literal> Columns</title>
</table>
</sect1>
+ <sect1 id="infoschema-column-column-usage">
+ <title><literal>column_column_usage</literal></title>
+
+ <para>
+ The view <literal>column_column_usage</literal> identifies all generated
+ columns that depend on another base column in the same table. Only tables
+ owned by a currently enabled role are included.
+ </para>
+
+ <table>
+ <title><literal>column_column_usage</literal> Columns</title>
+
+ <tgroup cols="3">
+ <thead>
+ <row>
+ <entry>Name</entry>
+ <entry>Data Type</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry><literal>table_catalog</literal></entry>
+ <entry><type>sql_identifier</type></entry>
+ <entry>Name of the database containing the table (always the current database)</entry>
+ </row>
+
+ <row>
+ <entry><literal>table_schema</literal></entry>
+ <entry><type>sql_identifier</type></entry>
+ <entry>Name of the schema containing the table</entry>
+ </row>
+
+ <row>
+ <entry><literal>table_name</literal></entry>
+ <entry><type>sql_identifier</type></entry>
+ <entry>Name of the table</entry>
+ </row>
+
+ <row>
+ <entry><literal>column_name</literal></entry>
+ <entry><type>sql_identifier</type></entry>
+ <entry>Name of the base column that a generated column depends on</entry>
+ </row>
+
+ <row>
+ <entry><literal>dependent_column</literal></entry>
+ <entry><type>sql_identifier</type></entry>
+ <entry>Name of the generated column</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+ </sect1>
+
<sect1 id="infoschema-column-domain-usage">
<title><literal>column_domain_usage</literal></title>
@@ -1648,13 +1704,19 @@ <title><literal>columns</literal> Columns</title>
<row>
<entry><literal>is_generated</literal></entry>
<entry><type>character_data</type></entry>
- <entry>Applies to a feature not available in <productname>PostgreSQL</productname></entry>
+ <entry>
+ If the column is a generated column, then <literal>ALWAYS</literal>,
+ else <literal>NEVER</literal>.
+ </entry>
</row>
<row>
<entry><literal>generation_expression</literal></entry>
<entry><type>character_data</type></entry>
- <entry>Applies to a feature not available in <productname>PostgreSQL</productname></entry>
+ <entry>
+ If the column is a generated column, then the generation expression,
+ else null.
+ </entry>
</row>
<row>
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index d66b860cbd..a0e1f78bfc 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -6450,7 +6450,7 @@ <title>Logical Replication Message Formats</title>
</listitem>
</varlistentry>
</variablelist>
- Next, the following message part appears for each column:
+ Next, the following message part appears for each column (except generated columns):
<variablelist>
<varlistentry>
<term>
@@ -6875,7 +6875,7 @@ <title>Logical Replication Message Formats</title>
</listitem>
</varlistentry>
</variablelist>
- Next, one of the following submessages appears for each column:
+ Next, one of the following submessages appears for each column (except generated columns):
<variablelist>
<varlistentry>
<term>
diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml
index 254d3ab8eb..5e2992ddac 100644
--- a/doc/src/sgml/ref/copy.sgml
+++ b/doc/src/sgml/ref/copy.sgml
@@ -103,7 +103,8 @@ <title>Parameters</title>
<listitem>
<para>
An optional list of columns to be copied. If no column list is
- specified, all columns of the table will be copied.
+ specified, all columns of the table except generated columns will be
+ copied.
</para>
</listitem>
</varlistentry>
diff --git a/doc/src/sgml/ref/create_foreign_table.sgml b/doc/src/sgml/ref/create_foreign_table.sgml
index 19eb5341e7..65ba3e3d37 100644
--- a/doc/src/sgml/ref/create_foreign_table.sgml
+++ b/doc/src/sgml/ref/create_foreign_table.sgml
@@ -42,7 +42,8 @@
{ NOT NULL |
NULL |
CHECK ( <replaceable class="parameter">expression</replaceable> ) [ NO INHERIT ] |
- DEFAULT <replaceable>default_expr</replaceable> }
+ DEFAULT <replaceable>default_expr</replaceable> |
+ GENERATED ALWAYS AS ( <replaceable>generation_expr</replaceable> ) STORED }
<phrase>and <replaceable class="parameter">table_constraint</replaceable> is:</phrase>
@@ -258,6 +259,30 @@ <title>Parameters</title>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>GENERATED ALWAYS AS ( <replaceable>generation_expr</replaceable> ) STORED</literal><indexterm><primary>generated column</primary></indexterm></term>
+ <listitem>
+ <para>
+ This clause creates the column as a <firstterm>generated
+ column</firstterm>. The column cannot be written to, and when read it
+ will be computed from the specified expression.
+ </para>
+
+ <para>
+ The keyword <literal>STORED</literal> is required to signify that the
+ column will be computed on write. (The computed value will be presented
+ to the foreign-data wrapper for storage and must be returned on
+ reading.)
+ </para>
+
+ <para>
+ The generation expression can refer to other columns in the table, but
+ not other generated columns. Any functions and operators used must be
+ immutable. References to other tables are not allowed.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><replaceable class="parameter">server_name</replaceable></term>
<listitem>
diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml
index e94fe2c3b6..3c8ee54859 100644
--- a/doc/src/sgml/ref/create_table.sgml
+++ b/doc/src/sgml/ref/create_table.sgml
@@ -62,6 +62,7 @@
NULL |
CHECK ( <replaceable class="parameter">expression</replaceable> ) [ NO INHERIT ] |
DEFAULT <replaceable>default_expr</replaceable> |
+ GENERATED ALWAYS AS ( <replaceable>generation_expr</replaceable> ) STORED |
GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( <replaceable>sequence_options</replaceable> ) ] |
UNIQUE <replaceable class="parameter">index_parameters</replaceable> |
PRIMARY KEY <replaceable class="parameter">index_parameters</replaceable> |
@@ -83,7 +84,7 @@
<phrase>and <replaceable class="parameter">like_option</replaceable> is:</phrase>
-{ INCLUDING | EXCLUDING } { COMMENTS | CONSTRAINTS | DEFAULTS | IDENTITY | INDEXES | STATISTICS | STORAGE | ALL }
+{ INCLUDING | EXCLUDING } { COMMENTS | CONSTRAINTS | DEFAULTS | GENERATED | IDENTITY | INDEXES | STATISTICS | STORAGE | ALL }
<phrase>and <replaceable class="parameter">partition_bound_spec</replaceable> is:</phrase>
@@ -627,6 +628,16 @@ <title>Parameters</title>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>INCLUDING GENERATED</literal></term>
+ <listitem>
+ <para>
+ Any generation expressions of copied column definitions will be
+ copied. By default, new columns will be regular base columns.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><literal>INCLUDING IDENTITY</literal></term>
<listitem>
@@ -797,6 +808,28 @@ <title>Parameters</title>
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>GENERATED ALWAYS AS ( <replaceable>generation_expr</replaceable> ) STORED</literal><indexterm><primary>generated column</primary></indexterm></term>
+ <listitem>
+ <para>
+ This clause creates the column as a <firstterm>generated
+ column</firstterm>. The column cannot be written to, and when read it
+ will be computed from the specified expression.
+ </para>
+
+ <para>
+ The keyword <literal>STORED</literal> is required to signify that the
+ column will be computed on write and will be stored on disk. default.
+ </para>
+
+ <para>
+ The generation expression can refer to other columns in the table, but
+ not other generated columns. Any functions and operators used must be
+ immutable. References to other tables are not allowed.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><literal>GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( <replaceable>sequence_options</replaceable> ) ]</literal></term>
<listitem>
@@ -2028,6 +2061,16 @@ <title>Multiple Identity Columns</title>
</para>
</refsect2>
+ <refsect2>
+ <title>Generated Columns</title>
+
+ <para>
+ The option <literal>STORED</literal> is not standard but is also used by
+ other SQL implementations. The SQL standard does not specify the storage
+ of generated columns.
+ </para>
+ </refsect2>
+
<refsect2>
<title><literal>LIKE</literal> Clause</title>
diff --git a/doc/src/sgml/ref/create_trigger.sgml b/doc/src/sgml/ref/create_trigger.sgml
index 6514ffc6ae..6456105de6 100644
--- a/doc/src/sgml/ref/create_trigger.sgml
+++ b/doc/src/sgml/ref/create_trigger.sgml
@@ -261,7 +261,9 @@ <title>Parameters</title>
UPDATE OF <replaceable>column_name1</replaceable> [, <replaceable>column_name2</replaceable> ... ]
</synopsis>
The trigger will only fire if at least one of the listed columns
- is mentioned as a target of the <command>UPDATE</command> command.
+ is mentioned as a target of the <command>UPDATE</command> command
+ or if one of the listed columns is a generated column that depends on a
+ column that is the target of the <command>UPDATE</command>.
</para>
<para>
diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml
index 3281f7cd33..40888a4d20 100644
--- a/doc/src/sgml/textsearch.sgml
+++ b/doc/src/sgml/textsearch.sgml
@@ -620,15 +620,17 @@ <title>Creating Indexes</title>
<para>
Another approach is to create a separate <type>tsvector</type> column
- to hold the output of <function>to_tsvector</function>. This example is a
+ to hold the output of <function>to_tsvector</function>. To keep this
+ column automatically up to date with its source data, use a stored
+ generated column. This example is a
concatenation of <literal>title</literal> and <literal>body</literal>,
using <function>coalesce</function> to ensure that one field will still be
indexed when the other is <literal>NULL</literal>:
<programlisting>
-ALTER TABLE pgweb ADD COLUMN textsearchable_index_col tsvector;
-UPDATE pgweb SET textsearchable_index_col =
- to_tsvector('english', coalesce(title,'') || ' ' || coalesce(body,''));
+ALTER TABLE pgweb
+ ADD COLUMN textsearchable_index_col tsvector
+ GENERATED ALWAYS AS (to_tsvector('english', coalesce(title, '') || ' ' || coalesce(body, ''))) STORED;
</programlisting>
Then we create a <acronym>GIN</acronym> index to speed up the search:
@@ -648,14 +650,6 @@ <title>Creating Indexes</title>
</programlisting>
</para>
- <para>
- When using a separate column to store the <type>tsvector</type>
- representation,
- it is necessary to create a trigger to keep the <type>tsvector</type>
- column current anytime <literal>title</literal> or <literal>body</literal> changes.
- <xref linkend="textsearch-update-triggers"/> explains how to do that.
- </para>
-
<para>
One advantage of the separate-column approach over an expression index
is that it is not necessary to explicitly specify the text search
@@ -1857,6 +1851,14 @@ <title>Triggers for Automatic Updates</title>
<secondary>for updating a derived tsvector column</secondary>
</indexterm>
+ <note>
+ <para>
+ The method described in this section has been obsoleted by the use of
+ stored generated columns, as described in <xref
+ linkend="textsearch-tables-index"/>.
+ </para>
+ </note>
+
<para>
When using a separate column to store the <type>tsvector</type> representation
of your documents, it is necessary to create a trigger to update the
diff --git a/doc/src/sgml/trigger.sgml b/doc/src/sgml/trigger.sgml
index be9c228448..67e1861e06 100644
--- a/doc/src/sgml/trigger.sgml
+++ b/doc/src/sgml/trigger.sgml
@@ -243,6 +243,24 @@ <title>Overview of Trigger Behavior</title>
operation, and so they can return <symbol>NULL</symbol>.
</para>
+ <para>
+ Some considerations apply for generated
+ columns.<indexterm><primary>generated column</primary><secondary>in
+ triggers</secondary></indexterm> Stored generated columns are computed after
+ <literal>BEFORE</literal> triggers and before <literal>AFTER</literal>
+ triggers. Therefore, the generated value can be inspected in
+ <literal>AFTER</literal> triggers. In <literal>BEFORE</literal> triggers,
+ the <literal>OLD</literal> row contains the old generated value, as one
+ would expect, but the <literal>NEW</literal> row does not yet contain the
+ new generated value and should not be accessed. In the C language
+ interface, the content of the column is undefined at this point; a
+ higher-level programming language should prevent access to a stored
+ generated column in the <literal>NEW</literal> row in a
+ <literal>BEFORE</literal> trigger. Changes to the value of a generated
+ column in a <literal>BEFORE</literal> trigger are ignored and will be
+ overwritten.
+ </para>
+
<para>
If more than one trigger is defined for the same event on the same
relation, the triggers will be fired in alphabetical order by
diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c
index 0158950a43..6bc4e4c036 100644
--- a/src/backend/access/common/tupdesc.c
+++ b/src/backend/access/common/tupdesc.c
@@ -131,6 +131,7 @@ CreateTupleDescCopy(TupleDesc tupdesc)
att->atthasdef = false;
att->atthasmissing = false;
att->attidentity = '\0';
+ att->attgenerated = '\0';
}
/* We can copy the tuple type identification, too */
@@ -165,6 +166,7 @@ CreateTupleDescCopyConstr(TupleDesc tupdesc)
TupleConstr *cpy = (TupleConstr *) palloc0(sizeof(TupleConstr));
cpy->has_not_null = constr->has_not_null;
+ cpy->has_generated_stored = constr->has_generated_stored;
if ((cpy->num_defval = constr->num_defval) > 0)
{
@@ -247,6 +249,7 @@ TupleDescCopy(TupleDesc dst, TupleDesc src)
att->atthasdef = false;
att->atthasmissing = false;
att->attidentity = '\0';
+ att->attgenerated = '\0';
}
dst->constr = NULL;
@@ -300,6 +303,7 @@ TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno,
dstAtt->atthasdef = false;
dstAtt->atthasmissing = false;
dstAtt->attidentity = '\0';
+ dstAtt->attgenerated = '\0';
}
/*
@@ -456,6 +460,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2)
return false;
if (attr1->attidentity != attr2->attidentity)
return false;
+ if (attr1->attgenerated != attr2->attgenerated)
+ return false;
if (attr1->attisdropped != attr2->attisdropped)
return false;
if (attr1->attislocal != attr2->attislocal)
@@ -476,6 +482,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2)
return false;
if (constr1->has_not_null != constr2->has_not_null)
return false;
+ if (constr1->has_generated_stored != constr2->has_generated_stored)
+ return false;
n = constr1->num_defval;
if (n != (int) constr2->num_defval)
return false;
@@ -638,6 +646,7 @@ TupleDescInitEntry(TupleDesc desc,
att->atthasdef = false;
att->atthasmissing = false;
att->attidentity = '\0';
+ att->attgenerated = '\0';
att->attisdropped = false;
att->attislocal = true;
att->attinhcount = 0;
@@ -697,6 +706,7 @@ TupleDescInitBuiltinEntry(TupleDesc desc,
att->atthasdef = false;
att->atthasmissing = false;
att->attidentity = '\0';
+ att->attgenerated = '\0';
att->attisdropped = false;
att->attislocal = true;
att->attinhcount = 0;
@@ -853,6 +863,7 @@ BuildDescForRelation(List *schema)
TupleConstr *constr = (TupleConstr *) palloc0(sizeof(TupleConstr));
constr->has_not_null = true;
+ constr->has_generated_stored = false;
constr->defval = NULL;
constr->missing = NULL;
constr->num_defval = 0;
diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c
index c7b5ff62f9..79020e12e7 100644
--- a/src/backend/catalog/heap.c
+++ b/src/backend/catalog/heap.c
@@ -70,6 +70,7 @@
#include "parser/parse_collate.h"
#include "parser/parse_expr.h"
#include "parser/parse_relation.h"
+#include "parser/parsetree.h"
#include "partitioning/partdesc.h"
#include "storage/lmgr.h"
#include "storage/predicate.h"
@@ -687,6 +688,7 @@ InsertPgAttributeTuple(Relation pg_attribute_rel,
values[Anum_pg_attribute_atthasdef - 1] = BoolGetDatum(new_attribute->atthasdef);
values[Anum_pg_attribute_atthasmissing - 1] = BoolGetDatum(new_attribute->atthasmissing);
values[Anum_pg_attribute_attidentity - 1] = CharGetDatum(new_attribute->attidentity);
+ values[Anum_pg_attribute_attgenerated - 1] = CharGetDatum(new_attribute->attgenerated);
values[Anum_pg_attribute_attisdropped - 1] = BoolGetDatum(new_attribute->attisdropped);
values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(new_attribute->attislocal);
values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(new_attribute->attinhcount);
@@ -1653,6 +1655,9 @@ RemoveAttributeById(Oid relid, AttrNumber attnum)
/* We don't want to keep stats for it anymore */
attStruct->attstattarget = 0;
+ /* Unset this so no one tries to look up the generation expression */
+ attStruct->attgenerated = '\0';
+
/*
* Change the column name to something that isn't likely to conflict
*/
@@ -2152,6 +2157,7 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
Relation attrrel;
HeapTuple atttup;
Form_pg_attribute attStruct;
+ char attgenerated;
Oid attrdefOid;
ObjectAddress colobject,
defobject;
@@ -2199,6 +2205,7 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
elog(ERROR, "cache lookup failed for attribute %d of relation %u",
attnum, RelationGetRelid(rel));
attStruct = (Form_pg_attribute) GETSTRUCT(atttup);
+ attgenerated = attStruct->attgenerated;
if (!attStruct->atthasdef)
{
Form_pg_attribute defAttStruct;
@@ -2219,7 +2226,7 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
valuesAtt[Anum_pg_attribute_atthasdef - 1] = true;
replacesAtt[Anum_pg_attribute_atthasdef - 1] = true;
- if (add_column_mode)
+ if (add_column_mode && !attgenerated)
{
expr2 = expression_planner(expr2);
estate = CreateExecutorState();
@@ -2281,7 +2288,26 @@ StoreAttrDefault(Relation rel, AttrNumber attnum,
/*
* Record dependencies on objects used in the expression, too.
*/
- recordDependencyOnExpr(&defobject, expr, NIL, DEPENDENCY_NORMAL);
+ if (attgenerated)
+ {
+ /*
+ * Generated column: Dropping anything that the generation expression
+ * refers to automatically drops the generated column.
+ */
+ recordDependencyOnSingleRelExpr(&colobject, expr, RelationGetRelid(rel),
+ DEPENDENCY_AUTO,
+ DEPENDENCY_AUTO, false);
+ }
+ else
+ {
+ /*
+ * Normal default: Dropping anything that the default refers to
+ * requires CASCADE and drops the default only.
+ */
+ recordDependencyOnSingleRelExpr(&defobject, expr, RelationGetRelid(rel),
+ DEPENDENCY_NORMAL,
+ DEPENDENCY_NORMAL, false);
+ }
/*
* Post creation hook for attribute defaults.
@@ -2539,12 +2565,14 @@ AddRelationNewConstraints(Relation rel,
expr = cookDefault(pstate, colDef->raw_default,
atp->atttypid, atp->atttypmod,
- NameStr(atp->attname));
+ NameStr(atp->attname),
+ atp->attgenerated);
/*
* If the expression is just a NULL constant, we do not bother to make
* an explicit pg_attrdef entry, since the default behavior is
- * equivalent.
+ * equivalent. This applies to column defaults, but not for generation
+ * expressions.
*
* Note a nonobvious property of this test: if the column is of a
* domain type, what we'll get is not a bare null Const but a
@@ -2553,7 +2581,9 @@ AddRelationNewConstraints(Relation rel,
* override any default that the domain might have.
*/
if (expr == NULL ||
- (IsA(expr, Const) &&((Const *) expr)->constisnull))
+ (!colDef->generated &&
+ IsA(expr, Const) &&
+ castNode(Const, expr)->constisnull))
continue;
/* If the DEFAULT is volatile we cannot use a missing value */
@@ -2910,6 +2940,46 @@ SetRelationNumChecks(Relation rel, int numchecks)
table_close(relrel, RowExclusiveLock);
}
+/*
+ * Check for references to generated columns
+ */
+static bool
+check_nested_generated_walker(Node *node, void *context)
+{
+ ParseState *pstate = context;
+
+ if (node == NULL)
+ return false;
+ else if (IsA(node, Var))
+ {
+ Var *var = (Var *) node;
+ Oid relid;
+ AttrNumber attnum;
+
+ relid = rt_fetch(var->varno, pstate->p_rtable)->relid;
+ attnum = var->varattno;
+
+ if (OidIsValid(relid) && AttributeNumberIsValid(attnum) && get_attgenerated(relid, attnum))
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot use generated column \"%s\" in column generation expression",
+ get_attname(relid, attnum, false)),
+ errdetail("A generated column cannot reference another generated column."),
+ parser_errposition(pstate, var->location)));
+
+ return false;
+ }
+ else
+ return expression_tree_walker(node, check_nested_generated_walker,
+ (void *) context);
+}
+
+static void
+check_nested_generated(ParseState *pstate, Node *node)
+{
+ check_nested_generated_walker(node, pstate);
+}
+
/*
* Take a raw default and convert it to a cooked format ready for
* storage.
@@ -2927,7 +2997,8 @@ cookDefault(ParseState *pstate,
Node *raw_default,
Oid atttypid,
int32 atttypmod,
- const char *attname)
+ const char *attname,
+ char attgenerated)
{
Node *expr;
@@ -2936,17 +3007,25 @@ cookDefault(ParseState *pstate,
/*
* Transform raw parsetree to executable expression.
*/
- expr = transformExpr(pstate, raw_default, EXPR_KIND_COLUMN_DEFAULT);
+ expr = transformExpr(pstate, raw_default, attgenerated ? EXPR_KIND_GENERATED_COLUMN : EXPR_KIND_COLUMN_DEFAULT);
/*
* Make sure default expr does not refer to any vars (we need this check
* since the pstate includes the target table).
*/
- if (contain_var_clause(expr))
+ if (!attgenerated && contain_var_clause(expr))
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
errmsg("cannot use column references in default expression")));
+ if (attgenerated)
+ check_nested_generated(pstate, expr);
+
+ if (attgenerated && contain_mutable_functions(expr))
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("generation expression is not immutable")));
+
/*
* transformExpr() should have already rejected subqueries, aggregates,
* window functions, and SRFs, based on the EXPR_KIND_ for a default
diff --git a/src/backend/catalog/information_schema.sql b/src/backend/catalog/information_schema.sql
index 94e482596f..16677e78d6 100644
--- a/src/backend/catalog/information_schema.sql
+++ b/src/backend/catalog/information_schema.sql
@@ -509,7 +509,29 @@ CREATE VIEW collation_character_set_applicability AS
* COLUMN_COLUMN_USAGE view
*/
--- feature not supported
+CREATE VIEW column_column_usage AS
+ SELECT CAST(current_database() AS sql_identifier) AS table_catalog,
+ CAST(n.nspname AS sql_identifier) AS table_schema,
+ CAST(c.relname AS sql_identifier) AS table_name,
+ CAST(ac.attname AS sql_identifier) AS column_name,
+ CAST(ad.attname AS sql_identifier) AS dependent_column
+
+ FROM pg_namespace n, pg_class c, pg_depend d,
+ pg_attribute ac, pg_attribute ad
+
+ WHERE n.oid = c.relnamespace
+ AND c.oid = ac.attrelid
+ AND c.oid = ad.attrelid
+ AND d.classid = 'pg_catalog.pg_class'::regclass
+ AND d.refclassid = 'pg_catalog.pg_class'::regclass
+ AND d.objid = d.refobjid
+ AND c.oid = d.objid
+ AND d.objsubid = ad.attnum
+ AND d.refobjsubid = ac.attnum
+ AND ad.attgenerated <> ''
+ AND pg_has_role(c.relowner, 'USAGE');
+
+GRANT SELECT ON column_column_usage TO PUBLIC;
/*
@@ -656,7 +678,7 @@ CREATE VIEW columns AS
CAST(c.relname AS sql_identifier) AS table_name,
CAST(a.attname AS sql_identifier) AS column_name,
CAST(a.attnum AS cardinal_number) AS ordinal_position,
- CAST(pg_get_expr(ad.adbin, ad.adrelid) AS character_data) AS column_default,
+ CAST(CASE WHEN a.attgenerated = '' THEN pg_get_expr(ad.adbin, ad.adrelid) END AS character_data) AS column_default,
CAST(CASE WHEN a.attnotnull OR (t.typtype = 'd' AND t.typnotnull) THEN 'NO' ELSE 'YES' END
AS yes_or_no)
AS is_nullable,
@@ -745,8 +767,8 @@ CREATE VIEW columns AS
CAST(seq.seqmin AS character_data) AS identity_minimum,
CAST(CASE WHEN seq.seqcycle THEN 'YES' ELSE 'NO' END AS yes_or_no) AS identity_cycle,
- CAST('NEVER' AS character_data) AS is_generated,
- CAST(null AS character_data) AS generation_expression,
+ CAST(CASE WHEN a.attgenerated <> '' THEN 'ALWAYS' ELSE 'NEVER' END AS character_data) AS is_generated,
+ CAST(CASE WHEN a.attgenerated <> '' THEN pg_get_expr(ad.adbin, ad.adrelid) END AS character_data) AS generation_expression,
CAST(CASE WHEN c.relkind IN ('r', 'p') OR
(c.relkind IN ('v', 'f') AND
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 705df8900b..27d3a012af 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -32,6 +32,7 @@
#include "commands/trigger.h"
#include "executor/execPartition.h"
#include "executor/executor.h"
+#include "executor/nodeModifyTable.h"
#include "executor/tuptable.h"
#include "foreign/fdwapi.h"
#include "libpq/libpq.h"
@@ -2922,6 +2923,21 @@ CopyFrom(CopyState cstate)
}
else
{
+ /*
+ * Compute stored generated columns
+ *
+ * Switch memory context so that the new tuple is in the same
+ * context as the old one.
+ */
+ if (resultRelInfo->ri_RelationDesc->rd_att->constr &&
+ resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored)
+ {
+ ExecComputeStoredGenerated(estate, slot);
+ MemoryContextSwitchTo(batchcontext);
+ tuple = ExecCopySlotHeapTuple(slot);
+ MemoryContextSwitchTo(oldcontext);
+ }
+
/*
* If the target is a plain table, check the constraints of
* the tuple.
@@ -3271,7 +3287,7 @@ BeginCopyFrom(ParseState *pstate,
fmgr_info(in_func_oid, &in_functions[attnum - 1]);
/* Get default info if needed */
- if (!list_member_int(cstate->attnumlist, attnum))
+ if (!list_member_int(cstate->attnumlist, attnum) && !att->attgenerated)
{
/* attribute is NOT to be copied from input */
/* use default value if one exists */
@@ -4876,6 +4892,11 @@ CopyAttributeOutCSV(CopyState cstate, char *string,
* or NIL if there was none (in which case we want all the non-dropped
* columns).
*
+ * We don't include generated columns in the generated full list and we don't
+ * allow them to be specified explicitly. They don't make sense for COPY
+ * FROM, but we could possibly allow them for COPY TO. But this way it's at
+ * least ensured that whatever we copy out can be copied back in.
+ *
* rel can be NULL ... it's only used for error reports.
*/
static List *
@@ -4893,6 +4914,8 @@ CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
{
if (TupleDescAttr(tupDesc, i)->attisdropped)
continue;
+ if (TupleDescAttr(tupDesc, i)->attgenerated)
+ continue;
attnums = lappend_int(attnums, i + 1);
}
}
@@ -4917,6 +4940,12 @@ CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist)
continue;
if (namestrcmp(&(att->attname), name) == 0)
{
+ if (att->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
+ errmsg("column \"%s\" is a generated column",
+ name),
+ errdetail("Generated columns cannot be used in COPY.")));
attnum = att->attnum;
break;
}
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 3183b2aaa1..80176cc9c3 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -760,6 +760,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
rawEnt->attnum = attnum;
rawEnt->raw_default = colDef->raw_default;
rawEnt->missingMode = false;
+ rawEnt->generated = colDef->generated;
rawDefaults = lappend(rawDefaults, rawEnt);
attr->atthasdef = true;
}
@@ -783,6 +784,9 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
if (colDef->identity)
attr->attidentity = colDef->identity;
+
+ if (colDef->generated)
+ attr->attgenerated = colDef->generated;
}
/*
@@ -863,6 +867,27 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
*/
rel = relation_open(relationId, AccessExclusiveLock);
+ /*
+ * Now add any newly specified column default and generation expressions
+ * to the new relation. These are passed to us in the form of raw
+ * parsetrees; we need to transform them to executable expression trees
+ * before they can be added. The most convenient way to do that is to
+ * apply the parser's transformExpr routine, but transformExpr doesn't
+ * work unless we have a pre-existing relation. So, the transformation has
+ * to be postponed to this final step of CREATE TABLE.
+ *
+ * This needs to be before processing the partitioning clauses because
+ * those could refer to generated columns.
+ */
+ if (rawDefaults)
+ AddRelationNewConstraints(rel, rawDefaults, NIL,
+ true, true, false, queryString);
+
+ /*
+ * Make column generation expressions visible for use by partitioning.
+ */
+ CommandCounterIncrement();
+
/* Process and store partition bound, if any. */
if (stmt->partbound)
{
@@ -1064,16 +1089,12 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId,
}
/*
- * Now add any newly specified column default values and CHECK constraints
- * to the new relation. These are passed to us in the form of raw
- * parsetrees; we need to transform them to executable expression trees
- * before they can be added. The most convenient way to do that is to
- * apply the parser's transformExpr routine, but transformExpr doesn't
- * work unless we have a pre-existing relation. So, the transformation has
- * to be postponed to this final step of CREATE TABLE.
+ * Now add any newly specified CHECK constraints to the new relation.
+ * Same as for defaults above, but these need to come after partitioning
+ * is set up.
*/
- if (rawDefaults || stmt->constraints)
- AddRelationNewConstraints(rel, rawDefaults, stmt->constraints,
+ if (stmt->constraints)
+ AddRelationNewConstraints(rel, NIL, stmt->constraints,
true, true, false, queryString);
ObjectAddressSet(address, RelationRelationId, relationId);
@@ -2232,6 +2253,13 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
def->is_not_null |= attribute->attnotnull;
/* Default and other constraints are handled below */
newattno[parent_attno - 1] = exist_attno;
+
+ /* Check for GENERATED conflicts */
+ if (def->generated != attribute->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("inherited column \"%s\" has a generation conflict",
+ attributeName)));
}
else
{
@@ -2249,6 +2277,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence,
def->storage = attribute->attstorage;
def->raw_default = NULL;
def->cooked_default = NULL;
+ def->generated = attribute->attgenerated;
def->collClause = NULL;
def->collOid = attribute->attcollation;
def->constraints = NIL;
@@ -5599,6 +5628,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
attribute.atthasdef = false;
attribute.atthasmissing = false;
attribute.attidentity = colDef->identity;
+ attribute.attgenerated = colDef->generated;
attribute.attisdropped = false;
attribute.attislocal = colDef->is_local;
attribute.attinhcount = colDef->inhcount;
@@ -5644,7 +5674,9 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel,
* DEFAULT value outside of the heap. This may be disabled inside
* AddRelationNewConstraints if the optimization cannot be applied.
*/
- rawEnt->missingMode = true;
+ rawEnt->missingMode = (!colDef->generated);
+
+ rawEnt->generated = colDef->generated;
/*
* This function is intended for CREATE TABLE, so it processes a
@@ -6225,6 +6257,12 @@ ATExecColumnDefault(Relation rel, const char *colName,
colName, RelationGetRelationName(rel)),
newDefault ? 0 : errhint("Use ALTER TABLE ... ALTER COLUMN ... DROP IDENTITY instead.")));
+ if (TupleDescAttr(tupdesc, attnum - 1)->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("column \"%s\" of relation \"%s\" is a generated column",
+ colName, RelationGetRelationName(rel))));
+
/*
* Remove any old default for the column. We use RESTRICT here for
* safety, but at present we do not expect anything to depend on the
@@ -6246,6 +6284,7 @@ ATExecColumnDefault(Relation rel, const char *colName,
rawEnt->attnum = attnum;
rawEnt->raw_default = newDefault;
rawEnt->missingMode = false;
+ rawEnt->generated = '\0';
/*
* This function is intended for CREATE TABLE, so it processes a
@@ -7546,6 +7585,32 @@ ATAddForeignKeyConstraint(List **wqueue, AlteredTableInfo *tab, Relation rel,
*/
checkFkeyPermissions(pkrel, pkattnum, numpks);
+ /*
+ * Check some things for generated columns.
+ */
+ for (i = 0; i < numfks; i++)
+ {
+ char attgenerated = TupleDescAttr(RelationGetDescr(rel), fkattnum[i] - 1)->attgenerated;
+
+ if (attgenerated)
+ {
+ /*
+ * Check restrictions on UPDATE/DELETE actions, per SQL standard
+ */
+ if (fkconstraint->fk_upd_action == FKCONSTR_ACTION_SETNULL ||
+ fkconstraint->fk_upd_action == FKCONSTR_ACTION_SETDEFAULT ||
+ fkconstraint->fk_upd_action == FKCONSTR_ACTION_CASCADE)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid ON UPDATE action for foreign key constraint containing generated column")));
+ if (fkconstraint->fk_del_action == FKCONSTR_ACTION_SETNULL ||
+ fkconstraint->fk_del_action == FKCONSTR_ACTION_SETDEFAULT)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("invalid ON DELETE action for foreign key constraint containing generated column")));
+ }
+ }
+
/*
* Look up the equality operators to use in the constraint.
*
@@ -9937,10 +10002,18 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel,
COERCE_IMPLICIT_CAST,
-1);
if (defaultexpr == NULL)
- ereport(ERROR,
- (errcode(ERRCODE_DATATYPE_MISMATCH),
- errmsg("default for column \"%s\" cannot be cast automatically to type %s",
- colName, format_type_be(targettype))));
+ {
+ if (attTup->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("generation expression for column \"%s\" cannot be cast automatically to type %s",
+ colName, format_type_be(targettype))));
+ else
+ ereport(ERROR,
+ (errcode(ERRCODE_DATATYPE_MISMATCH),
+ errmsg("default for column \"%s\" cannot be cast automatically to type %s",
+ colName, format_type_be(targettype))));
+ }
}
else
defaultexpr = NULL;
@@ -10016,6 +10089,21 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel,
*/
Assert(foundObject.objectSubId == 0);
}
+ else if (relKind == RELKIND_RELATION &&
+ foundObject.objectSubId != 0 &&
+ get_attgenerated(foundObject.objectId, foundObject.objectSubId))
+ {
+ /*
+ * Changing the type of a column that is used by a
+ * generated column is not allowed by SQL standard.
+ * It might be doable with some thinking and effort.
+ */
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot alter type of a column used by a generated column"),
+ errdetail("Column \"%s\" is used by generated column \"%s\".",
+ colName, get_attname(foundObject.objectId, foundObject.objectSubId, false))));
+ }
else
{
/* Not expecting any other direct dependencies... */
@@ -10160,7 +10248,8 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel,
/*
* Now scan for dependencies of this column on other things. The only
* thing we should find is the dependency on the column datatype, which we
- * want to remove, and possibly a collation dependency.
+ * want to remove, possibly a collation dependency, and dependencies on
+ * other columns if it is a generated column.
*/
ScanKeyInit(&key[0],
Anum_pg_depend_classid,
@@ -10181,15 +10270,26 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel,
while (HeapTupleIsValid(depTup = systable_getnext(scan)))
{
Form_pg_depend foundDep = (Form_pg_depend) GETSTRUCT(depTup);
+ ObjectAddress foundObject;
- if (foundDep->deptype != DEPENDENCY_NORMAL)
+ foundObject.classId = foundDep->refclassid;
+ foundObject.objectId = foundDep->refobjid;
+ foundObject.objectSubId = foundDep->refobjsubid;
+
+ if (foundDep->deptype != DEPENDENCY_NORMAL &&
+ foundDep->deptype != DEPENDENCY_AUTO)
elog(ERROR, "found unexpected dependency type '%c'",
foundDep->deptype);
if (!(foundDep->refclassid == TypeRelationId &&
foundDep->refobjid == attTup->atttypid) &&
!(foundDep->refclassid == CollationRelationId &&
- foundDep->refobjid == attTup->attcollation))
- elog(ERROR, "found unexpected dependency for column");
+ foundDep->refobjid == attTup->attcollation) &&
+ !(foundDep->refclassid == RelationRelationId &&
+ foundDep->refobjid == RelationGetRelid(rel) &&
+ foundDep->refobjsubid != 0)
+ )
+ elog(ERROR, "found unexpected dependency for column: %s",
+ getObjectDescription(&foundObject));
CatalogTupleDelete(depRel, &depTup->t_self);
}
@@ -14321,6 +14421,18 @@ ComputePartitionAttrs(ParseState *pstate, Relation rel, List *partParams, AttrNu
pelem->name),
parser_errposition(pstate, pelem->location)));
+ /*
+ * Generated columns cannot work: They are computed after BEFORE
+ * triggers, but partition routing is done before all triggers.
+ */
+ if (attform->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("cannot use generated column in partition key"),
+ errdetail("Column \"%s\" is a generated column.",
+ pelem->name),
+ parser_errposition(pstate, pelem->location)));
+
partattrs[attn] = attform->attnum;
atttype = attform->atttypid;
attcollation = attform->attcollation;
@@ -14408,6 +14520,25 @@ ComputePartitionAttrs(ParseState *pstate, Relation rel, List *partParams, AttrNu
errmsg("partition key expressions cannot contain system column references")));
}
+ /*
+ * Generated columns cannot work: They are computed after
+ * BEFORE triggers, but partition routing is done before all
+ * triggers.
+ */
+ i = -1;
+ while ((i = bms_next_member(expr_attrs, i)) >= 0)
+ {
+ AttrNumber attno = i + FirstLowInvalidHeapAttributeNumber;
+
+ if (TupleDescAttr(RelationGetDescr(rel), attno - 1)->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("cannot use generated column in partition key"),
+ errdetail("Column \"%s\" is a generated column.",
+ get_attname(RelationGetRelid(rel), attno, false)),
+ parser_errposition(pstate, pelem->location)));
+ }
+
/*
* While it is not exactly *wrong* for a partition expression
* to be a constant, it seems better to reject such keys.
diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c
index e03ffdde38..3ae2640abd 100644
--- a/src/backend/commands/trigger.c
+++ b/src/backend/commands/trigger.c
@@ -75,8 +75,9 @@ static int MyTriggerDepth = 0;
* they use, so we let them be duplicated. Be sure to update all if one needs
* to be changed, however.
*/
-#define GetUpdatedColumns(relinfo, estate) \
- (exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->updatedCols)
+#define GetAllUpdatedColumns(relinfo, estate) \
+ (bms_union(exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->updatedCols, \
+ exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->extraUpdatedCols))
/* Local function prototypes */
static void ConvertTriggerToFK(CreateTrigStmt *stmt, Oid funcoid);
@@ -640,6 +641,24 @@ CreateTrigger(CreateTrigStmt *stmt, const char *queryString,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("BEFORE trigger's WHEN condition cannot reference NEW system columns"),
parser_errposition(pstate, var->location)));
+ if (TRIGGER_FOR_BEFORE(tgtype) &&
+ var->varattno == 0 &&
+ RelationGetDescr(rel)->constr &&
+ RelationGetDescr(rel)->constr->has_generated_stored)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("BEFORE trigger's WHEN condition cannot reference NEW generated columns"),
+ errdetail("A whole-row reference is used and the table contains generated columns."),
+ parser_errposition(pstate, var->location)));
+ if (TRIGGER_FOR_BEFORE(tgtype) &&
+ var->varattno > 0 &&
+ TupleDescAttr(RelationGetDescr(rel), var->varattno - 1)->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_OBJECT_DEFINITION),
+ errmsg("BEFORE trigger's WHEN condition cannot reference NEW generated columns"),
+ errdetail("Column \"%s\" is a generated column.",
+ NameStr(TupleDescAttr(RelationGetDescr(rel), var->varattno - 1)->attname)),
+ parser_errposition(pstate, var->location)));
break;
default:
/* can't happen without add_missing_from, so just elog */
@@ -2931,7 +2950,7 @@ ExecBSUpdateTriggers(EState *estate, ResultRelInfo *relinfo)
CMD_UPDATE))
return;
- updatedCols = GetUpdatedColumns(relinfo, estate);
+ updatedCols = GetAllUpdatedColumns(relinfo, estate);
LocTriggerData.type = T_TriggerData;
LocTriggerData.tg_event = TRIGGER_EVENT_UPDATE |
@@ -2980,7 +2999,7 @@ ExecASUpdateTriggers(EState *estate, ResultRelInfo *relinfo,
if (trigdesc && trigdesc->trig_update_after_statement)
AfterTriggerSaveEvent(estate, relinfo, TRIGGER_EVENT_UPDATE,
false, NULL, NULL, NIL,
- GetUpdatedColumns(relinfo, estate),
+ GetAllUpdatedColumns(relinfo, estate),
transition_capture);
}
@@ -3049,7 +3068,7 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate,
LocTriggerData.tg_relation = relinfo->ri_RelationDesc;
LocTriggerData.tg_oldtable = NULL;
LocTriggerData.tg_newtable = NULL;
- updatedCols = GetUpdatedColumns(relinfo, estate);
+ updatedCols = GetAllUpdatedColumns(relinfo, estate);
for (i = 0; i < trigdesc->numtriggers; i++)
{
Trigger *trigger = &trigdesc->triggers[i];
@@ -3140,7 +3159,7 @@ ExecARUpdateTriggers(EState *estate, ResultRelInfo *relinfo,
AfterTriggerSaveEvent(estate, relinfo, TRIGGER_EVENT_UPDATE,
true, oldslot, newslot, recheckIndexes,
- GetUpdatedColumns(relinfo, estate),
+ GetAllUpdatedColumns(relinfo, estate),
transition_capture);
}
}
diff --git a/src/backend/commands/typecmds.c b/src/backend/commands/typecmds.c
index f94248dc95..7e6bcc5239 100644
--- a/src/backend/commands/typecmds.c
+++ b/src/backend/commands/typecmds.c
@@ -918,7 +918,8 @@ DefineDomain(CreateDomainStmt *stmt)
defaultExpr = cookDefault(pstate, constr->raw_expr,
basetypeoid,
basetypeMod,
- domainName);
+ domainName,
+ 0);
/*
* If the expression is just a NULL constant, we treat it
@@ -2228,7 +2229,8 @@ AlterDomainDefault(List *names, Node *defaultRaw)
defaultExpr = cookDefault(pstate, defaultRaw,
typTup->typbasetype,
typTup->typtypmod,
- NameStr(typTup->typname));
+ NameStr(typTup->typname),
+ 0);
/*
* If the expression is just a NULL constant, we treat the command
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 426686b6ef..03dcc7b820 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -102,7 +102,7 @@ static void EvalPlanQualStart(EPQState *epqstate, EState *parentestate,
Plan *planTree);
/*
- * Note that GetUpdatedColumns() also exists in commands/trigger.c. There does
+ * Note that GetAllUpdatedColumns() also exists in commands/trigger.c. There does
* not appear to be any good header to put it into, given the structures that
* it uses, so we let them be duplicated. Be sure to update both if one needs
* to be changed, however.
@@ -111,6 +111,9 @@ static void EvalPlanQualStart(EPQState *epqstate, EState *parentestate,
(exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->insertedCols)
#define GetUpdatedColumns(relinfo, estate) \
(exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->updatedCols)
+#define GetAllUpdatedColumns(relinfo, estate) \
+ (bms_union(exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->updatedCols, \
+ exec_rt_fetch((relinfo)->ri_RangeTableIndex, estate)->extraUpdatedCols))
/* end of local decls */
@@ -1316,6 +1319,7 @@ InitResultRelInfo(ResultRelInfo *resultRelInfo,
resultRelInfo->ri_FdwState = NULL;
resultRelInfo->ri_usesFdwDirectModify = false;
resultRelInfo->ri_ConstraintExprs = NULL;
+ resultRelInfo->ri_GeneratedExprs = NULL;
resultRelInfo->ri_junkFilter = NULL;
resultRelInfo->ri_projectReturning = NULL;
resultRelInfo->ri_onConflictArbiterIndexes = NIL;
@@ -2328,7 +2332,7 @@ ExecUpdateLockMode(EState *estate, ResultRelInfo *relinfo)
* been modified, then we can use a weaker lock, allowing for better
* concurrency.
*/
- updatedCols = GetUpdatedColumns(relinfo, estate);
+ updatedCols = GetAllUpdatedColumns(relinfo, estate);
keyCols = RelationGetIndexAttrBitmap(relinfo->ri_RelationDesc,
INDEX_ATTR_BITMAP_KEY);
diff --git a/src/backend/executor/execReplication.c b/src/backend/executor/execReplication.c
index d8b48c667c..f8f6463358 100644
--- a/src/backend/executor/execReplication.c
+++ b/src/backend/executor/execReplication.c
@@ -21,6 +21,7 @@
#include "access/xact.h"
#include "commands/trigger.h"
#include "executor/executor.h"
+#include "executor/nodeModifyTable.h"
#include "nodes/nodeFuncs.h"
#include "parser/parse_relation.h"
#include "parser/parsetree.h"
@@ -412,6 +413,11 @@ ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot)
{
List *recheckIndexes = NIL;
+ /* Compute stored generated columns */
+ if (rel->rd_att->constr &&
+ rel->rd_att->constr->has_generated_stored)
+ ExecComputeStoredGenerated(estate, slot);
+
/* Check the constraints of the tuple */
if (rel->rd_att->constr)
ExecConstraints(resultRelInfo, slot, estate);
@@ -473,6 +479,11 @@ ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate,
List *recheckIndexes = NIL;
bool update_indexes;
+ /* Compute stored generated columns */
+ if (rel->rd_att->constr &&
+ rel->rd_att->constr->has_generated_stored)
+ ExecComputeStoredGenerated(estate, slot);
+
/* Check the constraints of the tuple */
if (rel->rd_att->constr)
ExecConstraints(resultRelInfo, slot, estate);
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 7be0e7745a..61c4459f67 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -49,6 +49,7 @@
#include "foreign/fdwapi.h"
#include "miscadmin.h"
#include "nodes/nodeFuncs.h"
+#include "rewrite/rewriteHandler.h"
#include "storage/bufmgr.h"
#include "storage/lmgr.h"
#include "utils/builtins.h"
@@ -240,6 +241,89 @@ ExecCheckTIDVisible(EState *estate,
ExecClearTuple(tempSlot);
}
+/*
+ * Compute stored generated columns for a tuple
+ */
+void
+ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot)
+{
+ ResultRelInfo *resultRelInfo = estate->es_result_relation_info;
+ Relation rel = resultRelInfo->ri_RelationDesc;
+ TupleDesc tupdesc = RelationGetDescr(rel);
+ int natts = tupdesc->natts;
+ MemoryContext oldContext;
+ Datum *values;
+ bool *nulls;
+ bool *replaces;
+ HeapTuple oldtuple, newtuple;
+ bool should_free;
+
+ Assert(tupdesc->constr && tupdesc->constr->has_generated_stored);
+
+ /*
+ * If first time through for this result relation, build expression
+ * nodetrees for rel's stored generation expressions. Keep them in the
+ * per-query memory context so they'll survive throughout the query.
+ */
+ if (resultRelInfo->ri_GeneratedExprs == NULL)
+ {
+ oldContext = MemoryContextSwitchTo(estate->es_query_cxt);
+
+ resultRelInfo->ri_GeneratedExprs =
+ (ExprState **) palloc(natts * sizeof(ExprState *));
+
+ for (int i = 0; i < natts; i++)
+ {
+ if (TupleDescAttr(tupdesc, i)->attgenerated == ATTRIBUTE_GENERATED_STORED)
+ {
+ Expr *expr;
+
+ expr = (Expr *) build_column_default(rel, i + 1);
+ if (expr == NULL)
+ elog(ERROR, "no generation expression found for column number %d of table \"%s\"",
+ i + 1, RelationGetRelationName(rel));
+
+ resultRelInfo->ri_GeneratedExprs[i] = ExecPrepareExpr(expr, estate);
+ }
+ }
+
+ MemoryContextSwitchTo(oldContext);
+ }
+
+ oldContext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
+
+ values = palloc(sizeof(*values) * natts);
+ nulls = palloc(sizeof(*nulls) * natts);
+ replaces = palloc0(sizeof(*replaces) * natts);
+
+ for (int i = 0; i < natts; i++)
+ {
+ if (TupleDescAttr(tupdesc, i)->attgenerated == ATTRIBUTE_GENERATED_STORED)
+ {
+ ExprContext *econtext;
+ Datum val;
+ bool isnull;
+
+ econtext = GetPerTupleExprContext(estate);
+ econtext->ecxt_scantuple = slot;
+
+ val = ExecEvalExpr(resultRelInfo->ri_GeneratedExprs[i], econtext, &isnull);
+
+ values[i] = val;
+ nulls[i] = isnull;
+ replaces[i] = true;
+ }
+ }
+
+ oldtuple = ExecFetchSlotHeapTuple(slot, true, &should_free);
+ newtuple = heap_modify_tuple(oldtuple, tupdesc, values, nulls, replaces);
+ ExecForceStoreHeapTuple(newtuple, slot);
+ if (should_free)
+ heap_freetuple(oldtuple);
+
+ MemoryContextSwitchTo(oldContext);
+}
+
/* ----------------------------------------------------------------
* ExecInsert
*
@@ -297,6 +381,13 @@ ExecInsert(ModifyTableState *mtstate,
}
else if (resultRelInfo->ri_FdwRoutine)
{
+ /*
+ * Compute stored generated columns
+ */
+ if (resultRelationDesc->rd_att->constr &&
+ resultRelationDesc->rd_att->constr->has_generated_stored)
+ ExecComputeStoredGenerated(estate, slot);
+
/*
* insert into foreign table: let the FDW do it
*/
@@ -326,6 +417,13 @@ ExecInsert(ModifyTableState *mtstate,
*/
slot->tts_tableOid = RelationGetRelid(resultRelationDesc);
+ /*
+ * Compute stored generated columns
+ */
+ if (resultRelationDesc->rd_att->constr &&
+ resultRelationDesc->rd_att->constr->has_generated_stored)
+ ExecComputeStoredGenerated(estate, slot);
+
/*
* Check any RLS WITH CHECK policies.
*
@@ -964,6 +1062,13 @@ ExecUpdate(ModifyTableState *mtstate,
}
else if (resultRelInfo->ri_FdwRoutine)
{
+ /*
+ * Compute stored generated columns
+ */
+ if (resultRelationDesc->rd_att->constr &&
+ resultRelationDesc->rd_att->constr->has_generated_stored)
+ ExecComputeStoredGenerated(estate, slot);
+
/*
* update in foreign table: let the FDW do it
*/
@@ -994,6 +1099,13 @@ ExecUpdate(ModifyTableState *mtstate,
*/
slot->tts_tableOid = RelationGetRelid(resultRelationDesc);
+ /*
+ * Compute stored generated columns
+ */
+ if (resultRelationDesc->rd_att->constr &&
+ resultRelationDesc->rd_att->constr->has_generated_stored)
+ ExecComputeStoredGenerated(estate, slot);
+
/*
* Check any RLS UPDATE WITH CHECK policies
*
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 04cc15606d..2978e822dc 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -2390,6 +2390,7 @@ _copyRangeTblEntry(const RangeTblEntry *from)
COPY_BITMAPSET_FIELD(selectedCols);
COPY_BITMAPSET_FIELD(insertedCols);
COPY_BITMAPSET_FIELD(updatedCols);
+ COPY_BITMAPSET_FIELD(extraUpdatedCols);
COPY_NODE_FIELD(securityQuals);
return newnode;
@@ -2888,6 +2889,7 @@ _copyColumnDef(const ColumnDef *from)
COPY_NODE_FIELD(cooked_default);
COPY_SCALAR_FIELD(identity);
COPY_NODE_FIELD(identitySequence);
+ COPY_SCALAR_FIELD(generated);
COPY_NODE_FIELD(collClause);
COPY_SCALAR_FIELD(collOid);
COPY_NODE_FIELD(constraints);
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index 91c007ad5b..b11fcb9def 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -2564,6 +2564,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b)
COMPARE_NODE_FIELD(cooked_default);
COMPARE_SCALAR_FIELD(identity);
COMPARE_NODE_FIELD(identitySequence);
+ COMPARE_SCALAR_FIELD(generated);
COMPARE_NODE_FIELD(collClause);
COMPARE_SCALAR_FIELD(collOid);
COMPARE_NODE_FIELD(constraints);
@@ -2663,6 +2664,7 @@ _equalRangeTblEntry(const RangeTblEntry *a, const RangeTblEntry *b)
COMPARE_BITMAPSET_FIELD(selectedCols);
COMPARE_BITMAPSET_FIELD(insertedCols);
COMPARE_BITMAPSET_FIELD(updatedCols);
+ COMPARE_BITMAPSET_FIELD(extraUpdatedCols);
COMPARE_NODE_FIELD(securityQuals);
return true;
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 910a738c20..3282be0e4b 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -2792,6 +2792,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node)
WRITE_NODE_FIELD(cooked_default);
WRITE_CHAR_FIELD(identity);
WRITE_NODE_FIELD(identitySequence);
+ WRITE_CHAR_FIELD(generated);
WRITE_NODE_FIELD(collClause);
WRITE_OID_FIELD(collOid);
WRITE_NODE_FIELD(constraints);
@@ -3096,6 +3097,7 @@ _outRangeTblEntry(StringInfo str, const RangeTblEntry *node)
WRITE_BITMAPSET_FIELD(selectedCols);
WRITE_BITMAPSET_FIELD(insertedCols);
WRITE_BITMAPSET_FIELD(updatedCols);
+ WRITE_BITMAPSET_FIELD(extraUpdatedCols);
WRITE_NODE_FIELD(securityQuals);
}
@@ -3467,6 +3469,13 @@ _outConstraint(StringInfo str, const Constraint *node)
WRITE_CHAR_FIELD(generated_when);
break;
+ case CONSTR_GENERATED:
+ appendStringInfoString(str, "GENERATED");
+ WRITE_NODE_FIELD(raw_expr);
+ WRITE_STRING_FIELD(cooked_expr);
+ WRITE_CHAR_FIELD(generated_when);
+ break;
+
case CONSTR_CHECK:
appendStringInfoString(str, "CHECK");
WRITE_BOOL_FIELD(is_no_inherit);
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index eff98febf1..3b96492b36 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -1430,6 +1430,7 @@ _readRangeTblEntry(void)
READ_BITMAPSET_FIELD(selectedCols);
READ_BITMAPSET_FIELD(insertedCols);
READ_BITMAPSET_FIELD(updatedCols);
+ READ_BITMAPSET_FIELD(extraUpdatedCols);
READ_NODE_FIELD(securityQuals);
READ_DONE();
diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index 979c3c212f..cc222cb06c 100644
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -6570,8 +6570,9 @@ make_modifytable(PlannerInfo *root,
/*
* Try to modify the foreign table directly if (1) the FDW provides
- * callback functions needed for that, (2) there are no row-level
- * triggers on the foreign table, and (3) there are no WITH CHECK
+ * callback functions needed for that and (2) there are no local
+ * structures that need to be run for each modified row: row-level
+ * triggers on the foreign table, stored generated columns, WITH CHECK
* OPTIONs from parent views.
*/
direct_modify = false;
@@ -6581,7 +6582,8 @@ make_modifytable(PlannerInfo *root,
fdwroutine->IterateDirectModify != NULL &&
fdwroutine->EndDirectModify != NULL &&
withCheckOptionLists == NIL &&
- !has_row_triggers(subroot, rti, operation))
+ !has_row_triggers(subroot, rti, operation) &&
+ !has_stored_generated_columns(subroot, rti))
direct_modify = fdwroutine->PlanDirectModify(subroot, node, rti, i);
if (direct_modify)
direct_modify_plans = bms_add_member(direct_modify_plans, i);
diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c
index 1fa154e0cb..a811f6a547 100644
--- a/src/backend/optimizer/util/inherit.c
+++ b/src/backend/optimizer/util/inherit.c
@@ -272,6 +272,10 @@ expand_partitioned_rtentry(PlannerInfo *root, RangeTblEntry *parentrte,
if (!root->partColsUpdated)
root->partColsUpdated =
has_partition_attrs(parentrel, parentrte->updatedCols, NULL);
+ /*
+ * There shouldn't be any generated columns in the partition key.
+ */
+ Assert(!has_partition_attrs(parentrel, parentrte->extraUpdatedCols, NULL));
/* First expand the partitioned table itself. */
expand_single_inheritance_child(root, parentrte, parentRTindex, parentrel,
@@ -412,6 +416,8 @@ expand_single_inheritance_child(PlannerInfo *root, RangeTblEntry *parentrte,
appinfo->translated_vars);
childrte->updatedCols = translate_col_privs(parentrte->updatedCols,
appinfo->translated_vars);
+ childrte->extraUpdatedCols = translate_col_privs(parentrte->extraUpdatedCols,
+ appinfo->translated_vars);
}
}
diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c
index 702c4f89b8..335c859154 100644
--- a/src/backend/optimizer/util/plancat.c
+++ b/src/backend/optimizer/util/plancat.c
@@ -2071,6 +2071,25 @@ has_row_triggers(PlannerInfo *root, Index rti, CmdType event)
return result;
}
+bool
+has_stored_generated_columns(PlannerInfo *root, Index rti)
+{
+ RangeTblEntry *rte = planner_rt_fetch(rti, root);
+ Relation relation;
+ TupleDesc tupdesc;
+ bool result = false;
+
+ /* Assume we already have adequate lock */
+ relation = heap_open(rte->relid, NoLock);
+
+ tupdesc = RelationGetDescr(relation);
+ result = tupdesc->constr && tupdesc->constr->has_generated_stored;
+
+ heap_close(relation, NoLock);
+
+ return result;
+}
+
/*
* set_relation_partition_info
*
diff --git a/src/backend/parser/analyze.c b/src/backend/parser/analyze.c
index d6cdd16607..400558b552 100644
--- a/src/backend/parser/analyze.c
+++ b/src/backend/parser/analyze.c
@@ -2287,6 +2287,7 @@ transformUpdateTargetList(ParseState *pstate, List *origTlist)
RangeTblEntry *target_rte;
ListCell *orig_tl;
ListCell *tl;
+ TupleDesc tupdesc = pstate->p_target_relation->rd_att;
tlist = transformTargetList(pstate, origTlist,
EXPR_KIND_UPDATE_SOURCE);
@@ -2345,6 +2346,32 @@ transformUpdateTargetList(ParseState *pstate, List *origTlist)
if (orig_tl != NULL)
elog(ERROR, "UPDATE target count mismatch --- internal error");
+ /*
+ * Record in extraUpdatedCols generated columns referencing updated base
+ * columns.
+ */
+ if (tupdesc->constr &&
+ tupdesc->constr->has_generated_stored)
+ {
+ for (int i = 0; i < tupdesc->constr->num_defval; i++)
+ {
+ AttrDefault defval = tupdesc->constr->defval[i];
+ Node *expr;
+ Bitmapset *attrs_used = NULL;
+
+ /* skip if not generated column */
+ if (!TupleDescAttr(tupdesc, defval.adnum - 1)->attgenerated)
+ continue;
+
+ expr = stringToNode(defval.adbin);
+ pull_varattnos(expr, 1, &attrs_used);
+
+ if (bms_overlap(target_rte->updatedCols, attrs_used))
+ target_rte->extraUpdatedCols = bms_add_member(target_rte->extraUpdatedCols,
+ defval.adnum - FirstLowInvalidHeapAttributeNumber);
+ }
+ }
+
return tlist;
}
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 0a4822829a..f6d890cfef 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -680,7 +680,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
SAVEPOINT SCHEMA SCHEMAS SCROLL SEARCH SECOND_P SECURITY SELECT SEQUENCE SEQUENCES
SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW
SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P
- START STATEMENT STATISTICS STDIN STDOUT STORAGE STRICT_P STRIP_P
+ START STATEMENT STATISTICS STDIN STDOUT STORAGE STORED STRICT_P STRIP_P
SUBSCRIPTION SUBSTRING SUPPORT SYMMETRIC SYSID SYSTEM_P
TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN
@@ -3496,6 +3496,16 @@ ColConstraintElem:
n->location = @1;
$$ = (Node *)n;
}
+ | GENERATED generated_when AS '(' a_expr ')' STORED
+ {
+ Constraint *n = makeNode(Constraint);
+ n->contype = CONSTR_GENERATED;
+ n->generated_when = $2;
+ n->raw_expr = $5;
+ n->cooked_expr = NULL;
+ n->location = @1;
+ $$ = (Node *)n;
+ }
| REFERENCES qualified_name opt_column_list key_match key_actions
{
Constraint *n = makeNode(Constraint);
@@ -3586,6 +3596,7 @@ TableLikeOption:
| CONSTRAINTS { $$ = CREATE_TABLE_LIKE_CONSTRAINTS; }
| DEFAULTS { $$ = CREATE_TABLE_LIKE_DEFAULTS; }
| IDENTITY_P { $$ = CREATE_TABLE_LIKE_IDENTITY; }
+ | GENERATED { $$ = CREATE_TABLE_LIKE_GENERATED; }
| INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; }
| STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; }
| STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; }
@@ -15224,6 +15235,7 @@ unreserved_keyword:
| STDIN
| STDOUT
| STORAGE
+ | STORED
| STRICT_P
| STRIP_P
| SUBSCRIPTION
diff --git a/src/backend/parser/parse_agg.c b/src/backend/parser/parse_agg.c
index 183ea0f2c4..c745fcdd2b 100644
--- a/src/backend/parser/parse_agg.c
+++ b/src/backend/parser/parse_agg.c
@@ -520,6 +520,14 @@ check_agglevels_and_constraints(ParseState *pstate, Node *expr)
err = _("grouping operations are not allowed in partition key expressions");
break;
+ case EXPR_KIND_GENERATED_COLUMN:
+
+ if (isAgg)
+ err = _("aggregate functions are not allowed in column generation expressions");
+ else
+ err = _("grouping operations are not allowed in column generation expressions");
+
+ break;
case EXPR_KIND_CALL_ARGUMENT:
if (isAgg)
@@ -922,6 +930,9 @@ transformWindowFuncCall(ParseState *pstate, WindowFunc *wfunc,
case EXPR_KIND_COPY_WHERE:
err = _("window functions are not allowed in COPY FROM WHERE conditions");
break;
+ case EXPR_KIND_GENERATED_COLUMN:
+ err = _("window functions are not allowed in column generation expressions");
+ break;
/*
* There is intentionally no default: case here, so that the
diff --git a/src/backend/parser/parse_expr.c b/src/backend/parser/parse_expr.c
index e559353529..0c88d0b2ee 100644
--- a/src/backend/parser/parse_expr.c
+++ b/src/backend/parser/parse_expr.c
@@ -1854,6 +1854,9 @@ transformSubLink(ParseState *pstate, SubLink *sublink)
case EXPR_KIND_COPY_WHERE:
err = _("cannot use subquery in COPY FROM WHERE condition");
break;
+ case EXPR_KIND_GENERATED_COLUMN:
+ err = _("cannot use subquery in column generation expression");
+ break;
/*
* There is intentionally no default: case here, so that the
@@ -3484,6 +3487,8 @@ ParseExprKindName(ParseExprKind exprKind)
return "CALL";
case EXPR_KIND_COPY_WHERE:
return "WHERE";
+ case EXPR_KIND_GENERATED_COLUMN:
+ return "GENERATED AS";
/*
* There is intentionally no default: case here, so that the
diff --git a/src/backend/parser/parse_func.c b/src/backend/parser/parse_func.c
index b8447771bd..752cf1b315 100644
--- a/src/backend/parser/parse_func.c
+++ b/src/backend/parser/parse_func.c
@@ -2526,6 +2526,9 @@ check_srf_call_placement(ParseState *pstate, Node *last_srf, int location)
case EXPR_KIND_COPY_WHERE:
err = _("set-returning functions are not allowed in COPY FROM WHERE conditions");
break;
+ case EXPR_KIND_GENERATED_COLUMN:
+ err = _("set-returning functions are not allowed in column generation expressions");
+ break;
/*
* There is intentionally no default: case here, so that the
diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c
index f3b6d193aa..0640d11fac 100644
--- a/src/backend/parser/parse_relation.c
+++ b/src/backend/parser/parse_relation.c
@@ -731,6 +731,17 @@ scanRTEForColumn(ParseState *pstate, RangeTblEntry *rte, const char *colname,
colname),
parser_errposition(pstate, location)));
+ /*
+ * In generated column, no system column is allowed except tableOid.
+ */
+ if (pstate->p_expr_kind == EXPR_KIND_GENERATED_COLUMN &&
+ attnum < InvalidAttrNumber && attnum != TableOidAttributeNumber)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
+ errmsg("cannot use system column \"%s\" in column generation expression",
+ colname),
+ parser_errposition(pstate, location)));
+
if (attnum != InvalidAttrNumber)
{
/* now check to see if column actually is defined */
@@ -1257,6 +1268,7 @@ addRangeTableEntry(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
@@ -1328,6 +1340,7 @@ addRangeTableEntryForRelation(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
@@ -1407,6 +1420,7 @@ addRangeTableEntryForSubquery(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
@@ -1670,6 +1684,7 @@ addRangeTableEntryForFunction(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
@@ -1733,6 +1748,7 @@ addRangeTableEntryForTableFunc(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
@@ -1811,6 +1827,7 @@ addRangeTableEntryForValues(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
@@ -1881,6 +1898,7 @@ addRangeTableEntryForJoin(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
@@ -1983,6 +2001,7 @@ addRangeTableEntryForCTE(ParseState *pstate,
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
/*
* Add completed RTE to pstate's range table list, but not to join list
diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c
index b4ec96d6d6..f8b4910509 100644
--- a/src/backend/parser/parse_utilcmd.c
+++ b/src/backend/parser/parse_utilcmd.c
@@ -502,6 +502,7 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column)
bool saw_nullable;
bool saw_default;
bool saw_identity;
+ bool saw_generated;
ListCell *clist;
cxt->columns = lappend(cxt->columns, column);
@@ -609,6 +610,7 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column)
saw_nullable = false;
saw_default = false;
saw_identity = false;
+ saw_generated = false;
foreach(clist, column->constraints)
{
@@ -689,6 +691,29 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column)
break;
}
+ case CONSTR_GENERATED:
+ if (cxt->ofType)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("generated columns are not supported on typed tables")));
+ if (cxt->partbound)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("generated columns are not supported on partitions")));
+
+ if (saw_generated)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("multiple generation clauses specified for column \"%s\" of table \"%s\"",
+ column->colname, cxt->relation->relname),
+ parser_errposition(cxt->pstate,
+ constraint->location)));
+ column->generated = ATTRIBUTE_GENERATED_STORED;
+ column->raw_default = constraint->raw_expr;
+ Assert(constraint->cooked_expr == NULL);
+ saw_generated = true;
+ break;
+
case CONSTR_CHECK:
cxt->ckconstraints = lappend(cxt->ckconstraints, constraint);
break;
@@ -755,6 +780,22 @@ transformColumnDefinition(CreateStmtContext *cxt, ColumnDef *column)
column->colname, cxt->relation->relname),
parser_errposition(cxt->pstate,
constraint->location)));
+
+ if (saw_default && saw_generated)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("both default and generation expression specified for column \"%s\" of table \"%s\"",
+ column->colname, cxt->relation->relname),
+ parser_errposition(cxt->pstate,
+ constraint->location)));
+
+ if (saw_identity && saw_generated)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("both identity and generation expression specified for column \"%s\" of table \"%s\"",
+ column->colname, cxt->relation->relname),
+ parser_errposition(cxt->pstate,
+ constraint->location)));
}
/*
@@ -983,11 +1024,13 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla
* Copy default, if present and the default has been requested
*/
if (attribute->atthasdef &&
- (table_like_clause->options & CREATE_TABLE_LIKE_DEFAULTS))
+ (table_like_clause->options & CREATE_TABLE_LIKE_DEFAULTS ||
+ table_like_clause->options & CREATE_TABLE_LIKE_GENERATED))
{
Node *this_default = NULL;
AttrDefault *attrdef;
int i;
+ bool found_whole_row;
/* Find default in constraint structure */
Assert(constr != NULL);
@@ -1002,12 +1045,27 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla
}
Assert(this_default != NULL);
+ def->cooked_default = map_variable_attnos(this_default,
+ 1, 0,
+ attmap, tupleDesc->natts,
+ InvalidOid, &found_whole_row);
+
/*
- * If default expr could contain any vars, we'd need to fix 'em,
- * but it can't; so default is ready to apply to child.
+ * Prevent this for the same reason as for constraints below.
+ * Note that defaults cannot contain any vars, so it's OK that the
+ * error message refers to generated columns.
*/
+ if (found_whole_row)
+ ereport(ERROR,
+ (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+ errmsg("cannot convert whole-row table reference"),
+ errdetail("Generation expression for column \"%s\" contains a whole-row reference to table \"%s\".",
+ attributeName,
+ RelationGetRelationName(relation))));
- def->cooked_default = this_default;
+ if (attribute->attgenerated &&
+ (table_like_clause->options & CREATE_TABLE_LIKE_GENERATED))
+ def->generated = attribute->attgenerated;
}
/*
diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c
index dffb6cd9fd..0411963f93 100644
--- a/src/backend/replication/logical/proto.c
+++ b/src/backend/replication/logical/proto.c
@@ -453,7 +453,7 @@ logicalrep_write_tuple(StringInfo out, Relation rel, HeapTuple tuple)
for (i = 0; i < desc->natts; i++)
{
- if (TupleDescAttr(desc, i)->attisdropped)
+ if (TupleDescAttr(desc, i)->attisdropped || TupleDescAttr(desc, i)->attgenerated)
continue;
nliveatts++;
}
@@ -473,8 +473,7 @@ logicalrep_write_tuple(StringInfo out, Relation rel, HeapTuple tuple)
Form_pg_attribute att = TupleDescAttr(desc, i);
char *outputstr;
- /* skip dropped columns */
- if (att->attisdropped)
+ if (att->attisdropped || att->attgenerated)
continue;
if (isnull[i])
@@ -573,7 +572,7 @@ logicalrep_write_attrs(StringInfo out, Relation rel)
/* send number of live attributes */
for (i = 0; i < desc->natts; i++)
{
- if (TupleDescAttr(desc, i)->attisdropped)
+ if (TupleDescAttr(desc, i)->attisdropped || TupleDescAttr(desc, i)->attgenerated)
continue;
nliveatts++;
}
@@ -591,7 +590,7 @@ logicalrep_write_attrs(StringInfo out, Relation rel)
Form_pg_attribute att = TupleDescAttr(desc, i);
uint8 flags = 0;
- if (att->attisdropped)
+ if (att->attisdropped || att->attgenerated)
continue;
/* REPLICA IDENTITY FULL means all columns are sent as part of key. */
diff --git a/src/backend/replication/logical/relation.c b/src/backend/replication/logical/relation.c
index 1d918d2c42..5aee4b80e6 100644
--- a/src/backend/replication/logical/relation.c
+++ b/src/backend/replication/logical/relation.c
@@ -276,7 +276,7 @@ logicalrep_rel_open(LogicalRepRelId remoteid, LOCKMODE lockmode)
int attnum;
Form_pg_attribute attr = TupleDescAttr(desc, i);
- if (attr->attisdropped)
+ if (attr->attisdropped || attr->attgenerated)
{
entry->attrmap[i] = -1;
continue;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 28f5fc23aa..7881079e96 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -697,10 +697,12 @@ fetch_remote_table_info(char *nspname, char *relname,
" LEFT JOIN pg_catalog.pg_index i"
" ON (i.indexrelid = pg_get_replica_identity_index(%u))"
" WHERE a.attnum > 0::pg_catalog.int2"
- " AND NOT a.attisdropped"
+ " AND NOT a.attisdropped %s"
" AND a.attrelid = %u"
" ORDER BY a.attnum",
- lrel->remoteid, lrel->remoteid);
+ lrel->remoteid,
+ (walrcv_server_version(wrconn) >= 120000 ? "AND a.attgenerated = ''" : ""),
+ lrel->remoteid);
res = walrcv_exec(wrconn, cmd.data, 4, attrRow);
if (res->status != WALRCV_OK_TUPLES)
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 52a5090b69..43edfef089 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -236,7 +236,7 @@ slot_fill_defaults(LogicalRepRelMapEntry *rel, EState *estate,
{
Expr *defexpr;
- if (TupleDescAttr(desc, attnum)->attisdropped)
+ if (TupleDescAttr(desc, attnum)->attisdropped || TupleDescAttr(desc, attnum)->attgenerated)
continue;
if (rel->attrmap[attnum] >= 0)
diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c
index 5511957516..bf64c8e4a4 100644
--- a/src/backend/replication/pgoutput/pgoutput.c
+++ b/src/backend/replication/pgoutput/pgoutput.c
@@ -276,7 +276,7 @@ maybe_send_schema(LogicalDecodingContext *ctx,
{
Form_pg_attribute att = TupleDescAttr(desc, i);
- if (att->attisdropped)
+ if (att->attisdropped || att->attgenerated)
continue;
if (att->atttypid < FirstNormalObjectId)
diff --git a/src/backend/rewrite/rewriteHandler.c b/src/backend/rewrite/rewriteHandler.c
index 4fc50c89b9..39080776b0 100644
--- a/src/backend/rewrite/rewriteHandler.c
+++ b/src/backend/rewrite/rewriteHandler.c
@@ -818,6 +818,13 @@ rewriteTargetListIU(List *targetList,
if (att_tup->attidentity == ATTRIBUTE_IDENTITY_BY_DEFAULT && override == OVERRIDING_USER_VALUE)
apply_default = true;
+
+ if (att_tup->attgenerated && !apply_default)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("cannot insert into column \"%s\"", NameStr(att_tup->attname)),
+ errdetail("Column \"%s\" is a generated column.",
+ NameStr(att_tup->attname))));
}
if (commandType == CMD_UPDATE)
@@ -828,9 +835,23 @@ rewriteTargetListIU(List *targetList,
errmsg("column \"%s\" can only be updated to DEFAULT", NameStr(att_tup->attname)),
errdetail("Column \"%s\" is an identity column defined as GENERATED ALWAYS.",
NameStr(att_tup->attname))));
+
+ if (att_tup->attgenerated && new_tle && !apply_default)
+ ereport(ERROR,
+ (errcode(ERRCODE_SYNTAX_ERROR),
+ errmsg("column \"%s\" can only be updated to DEFAULT", NameStr(att_tup->attname)),
+ errdetail("Column \"%s\" is a generated column.",
+ NameStr(att_tup->attname))));
}
- if (apply_default)
+ if (att_tup->attgenerated)
+ {
+ /*
+ * stored generated column will be fixed in executor
+ */
+ new_tle = NULL;
+ }
+ else if (apply_default)
{
Node *new_expr;
@@ -1137,13 +1158,12 @@ build_column_default(Relation rel, int attrno)
}
}
- if (expr == NULL)
- {
- /*
- * No per-column default, so look for a default for the type itself.
- */
+ /*
+ * No per-column default, so look for a default for the type itself. But
+ * not for generated columns.
+ */
+ if (expr == NULL && !att_tup->attgenerated)
expr = get_typdefault(atttype);
- }
if (expr == NULL)
return NULL; /* No default anywhere */
@@ -1720,12 +1740,14 @@ ApplyRetrieveRule(Query *parsetree,
subrte->selectedCols = rte->selectedCols;
subrte->insertedCols = rte->insertedCols;
subrte->updatedCols = rte->updatedCols;
+ subrte->extraUpdatedCols = rte->extraUpdatedCols;
rte->requiredPerms = 0; /* no permission check on subquery itself */
rte->checkAsUser = InvalidOid;
rte->selectedCols = NULL;
rte->insertedCols = NULL;
rte->updatedCols = NULL;
+ rte->extraUpdatedCols = NULL;
return parsetree;
}
diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c
index 59e6bcd856..10895567c0 100644
--- a/src/backend/utils/cache/lsyscache.c
+++ b/src/backend/utils/cache/lsyscache.c
@@ -821,6 +821,39 @@ get_attnum(Oid relid, const char *attname)
return InvalidAttrNumber;
}
+/*
+ * get_attgenerated
+ *
+ * Given the relation id and the attribute name,
+ * return the "attgenerated" field from the attribute relation.
+ *
+ * Errors if not found.
+ *
+ * Since not generated is represented by '\0', this can also be used as a
+ * Boolean test.
+ */
+char
+get_attgenerated(Oid relid, AttrNumber attnum)
+{
+ HeapTuple tp;
+
+ tp = SearchSysCache2(ATTNUM,
+ ObjectIdGetDatum(relid),
+ Int16GetDatum(attnum));
+ if (HeapTupleIsValid(tp))
+ {
+ Form_pg_attribute att_tup = (Form_pg_attribute) GETSTRUCT(tp);
+ char result;
+
+ result = att_tup->attgenerated;
+ ReleaseSysCache(tp);
+ return result;
+ }
+ else
+ elog(ERROR, "cache lookup failed for attribute %d of relation %u",
+ attnum, relid);
+}
+
/*
* get_atttype
*
diff --git a/src/backend/utils/cache/partcache.c b/src/backend/utils/cache/partcache.c
index 2b55f25e75..8f43d682cf 100644
--- a/src/backend/utils/cache/partcache.c
+++ b/src/backend/utils/cache/partcache.c
@@ -27,6 +27,7 @@
#include "nodes/nodeFuncs.h"
#include "optimizer/optimizer.h"
#include "partitioning/partbounds.h"
+#include "rewrite/rewriteHandler.h"
#include "utils/builtins.h"
#include "utils/datum.h"
#include "utils/lsyscache.h"
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index 84609e0725..4557164b00 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -515,6 +515,7 @@ RelationBuildTupleDesc(Relation relation)
constr = (TupleConstr *) MemoryContextAlloc(CacheMemoryContext,
sizeof(TupleConstr));
constr->has_not_null = false;
+ constr->has_generated_stored = false;
/*
* Form a scan key that selects only user attributes (attnum > 0).
@@ -567,6 +568,8 @@ RelationBuildTupleDesc(Relation relation)
/* Update constraint/default info */
if (attp->attnotnull)
constr->has_not_null = true;
+ if (attp->attgenerated == ATTRIBUTE_GENERATED_STORED)
+ constr->has_generated_stored = true;
/* If the column has a default, fill it into the attrdef array */
if (attp->atthasdef)
@@ -3281,6 +3284,7 @@ RelationBuildLocalRelation(const char *relname,
Form_pg_attribute datt = TupleDescAttr(rel->rd_att, i);
datt->attidentity = satt->attidentity;
+ datt->attgenerated = satt->attgenerated;
datt->attnotnull = satt->attnotnull;
has_not_null |= satt->attnotnull;
}
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 63699932c1..2a9e8538c6 100644
--- a/src/bin/pg_dump/pg_dump.c
+++ b/src/bin/pg_dump/pg_dump.c
@@ -2051,6 +2051,11 @@ dumpTableData_insert(Archive *fout, void *dcontext)
{
if (field > 0)
archputs(", ", fout);
+ if (tbinfo->attgenerated[field])
+ {
+ archputs("DEFAULT", fout);
+ continue;
+ }
if (PQgetisnull(res, tuple, field))
{
archputs("NULL", fout);
@@ -8219,6 +8224,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
int i_attnotnull;
int i_atthasdef;
int i_attidentity;
+ int i_attgenerated;
int i_attisdropped;
int i_attlen;
int i_attalign;
@@ -8272,6 +8278,13 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
"a.attislocal,\n"
"pg_catalog.format_type(t.oid, a.atttypmod) AS atttypname,\n");
+ if (fout->remoteVersion >= 120000)
+ appendPQExpBuffer(q,
+ "a.attgenerated,\n");
+ else
+ appendPQExpBuffer(q,
+ "'' AS attgenerated,\n");
+
if (fout->remoteVersion >= 110000)
appendPQExpBuffer(q,
"CASE WHEN a.atthasmissing AND NOT a.attisdropped "
@@ -8344,6 +8357,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
i_attnotnull = PQfnumber(res, "attnotnull");
i_atthasdef = PQfnumber(res, "atthasdef");
i_attidentity = PQfnumber(res, "attidentity");
+ i_attgenerated = PQfnumber(res, "attgenerated");
i_attisdropped = PQfnumber(res, "attisdropped");
i_attlen = PQfnumber(res, "attlen");
i_attalign = PQfnumber(res, "attalign");
@@ -8361,6 +8375,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
tbinfo->attstorage = (char *) pg_malloc(ntups * sizeof(char));
tbinfo->typstorage = (char *) pg_malloc(ntups * sizeof(char));
tbinfo->attidentity = (char *) pg_malloc(ntups * sizeof(char));
+ tbinfo->attgenerated = (char *) pg_malloc(ntups * sizeof(char));
tbinfo->attisdropped = (bool *) pg_malloc(ntups * sizeof(bool));
tbinfo->attlen = (int *) pg_malloc(ntups * sizeof(int));
tbinfo->attalign = (char *) pg_malloc(ntups * sizeof(char));
@@ -8387,6 +8402,7 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables)
tbinfo->attstorage[j] = *(PQgetvalue(res, j, i_attstorage));
tbinfo->typstorage[j] = *(PQgetvalue(res, j, i_typstorage));
tbinfo->attidentity[j] = *(PQgetvalue(res, j, i_attidentity));
+ tbinfo->attgenerated[j] = *(PQgetvalue(res, j, i_attgenerated));
tbinfo->needs_override = tbinfo->needs_override || (tbinfo->attidentity[j] == ATTRIBUTE_IDENTITY_ALWAYS);
tbinfo->attisdropped[j] = (PQgetvalue(res, j, i_attisdropped)[0] == 't');
tbinfo->attlen[j] = atoi(PQgetvalue(res, j, i_attlen));
@@ -15708,6 +15724,20 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo)
tbinfo->atttypnames[j]);
}
+ if (has_default)
+ {
+ if (tbinfo->attgenerated[j] == ATTRIBUTE_GENERATED_STORED)
+ appendPQExpBuffer(q, " GENERATED ALWAYS AS (%s) STORED",
+ tbinfo->attrdefs[j]->adef_expr);
+ else
+ appendPQExpBuffer(q, " DEFAULT %s",
+ tbinfo->attrdefs[j]->adef_expr);
+ }
+
+
+ if (has_notnull)
+ appendPQExpBufferStr(q, " NOT NULL");
+
/* Add collation if not default for the type */
if (OidIsValid(tbinfo->attcollation[j]))
{
@@ -15718,13 +15748,6 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo)
appendPQExpBuffer(q, " COLLATE %s",
fmtQualifiedDumpable(coll));
}
-
- if (has_default)
- appendPQExpBuffer(q, " DEFAULT %s",
- tbinfo->attrdefs[j]->adef_expr);
-
- if (has_notnull)
- appendPQExpBufferStr(q, " NOT NULL");
}
}
@@ -18303,6 +18326,7 @@ fmtCopyColumnList(const TableInfo *ti, PQExpBuffer buffer)
int numatts = ti->numatts;
char **attnames = ti->attnames;
bool *attisdropped = ti->attisdropped;
+ char *attgenerated = ti->attgenerated;
bool needComma;
int i;
@@ -18312,6 +18336,8 @@ fmtCopyColumnList(const TableInfo *ti, PQExpBuffer buffer)
{
if (attisdropped[i])
continue;
+ if (attgenerated[i])
+ continue;
if (needComma)
appendPQExpBufferStr(buffer, ", ");
appendPQExpBufferStr(buffer, fmtId(attnames[i]));
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 2e1b90acd0..a72e3eb27c 100644
--- a/src/bin/pg_dump/pg_dump.h
+++ b/src/bin/pg_dump/pg_dump.h
@@ -310,6 +310,7 @@ typedef struct _tableInfo
char *typstorage; /* type storage scheme */
bool *attisdropped; /* true if attr is dropped; don't dump it */
char *attidentity;
+ char *attgenerated;
int *attlen; /* attribute length, used by binary_upgrade */
char *attalign; /* attribute align, used by binary_upgrade */
bool *attislocal; /* true if attr has local definition */
diff --git a/src/bin/pg_dump/pg_dump_sort.c b/src/bin/pg_dump/pg_dump_sort.c
index bb128c89f3..243dca7264 100644
--- a/src/bin/pg_dump/pg_dump_sort.c
+++ b/src/bin/pg_dump/pg_dump_sort.c
@@ -1108,6 +1108,16 @@ repairDependencyLoop(DumpableObject **loop,
}
}
+ /* Loop of table with itself, happens with generated columns */
+ if (nLoop == 1)
+ {
+ if (loop[0]->objType == DO_TABLE)
+ {
+ removeObjectDependency(loop[0], loop[0]->dumpId);
+ return;
+ }
+ }
+
/*
* If all the objects are TABLE_DATA items, what we must have is a
* circular set of foreign key constraints (or a single self-referential
diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index de6895122e..a69375056d 100644
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -2392,6 +2392,23 @@
unlike => { exclude_dump_test_schema => 1, },
},
+ 'CREATE TABLE test_table_generated' => {
+ create_order => 3,
+ create_sql => 'CREATE TABLE dump_test.test_table_generated (
+ col1 int primary key,
+ col2 int generated always as (col1 * 2) stored
+ );',
+ regexp => qr/^
+ \QCREATE TABLE dump_test.test_table_generated (\E\n
+ \s+\Qcol1 integer NOT NULL,\E\n
+ \s+\Qcol2 integer GENERATED ALWAYS AS ((col1 * 2)) STORED\E\n
+ \);
+ /xms,
+ like =>
+ { %full_runs, %dump_test_schema_runs, section_pre_data => 1, },
+ unlike => { exclude_dump_test_schema => 1, },
+ },
+
'CREATE TABLE table_with_stats' => {
create_order => 98,
create_sql => 'CREATE TABLE dump_test.table_index_stats (
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index fd8ebee8cd..1c9f20a576 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -1462,6 +1462,7 @@ describeOneTableDetails(const char *schemaname,
attnotnull_col = -1,
attcoll_col = -1,
attidentity_col = -1,
+ attgenerated_col = -1,
isindexkey_col = -1,
indexdef_col = -1,
fdwopts_col = -1,
@@ -1813,8 +1814,9 @@ describeOneTableDetails(const char *schemaname,
if (show_column_details)
{
+ /* use "pretty" mode for expression to avoid excessive parentheses */
appendPQExpBufferStr(&buf,
- ",\n (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid) for 128)"
+ ",\n (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128)"
"\n FROM pg_catalog.pg_attrdef d"
"\n WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef)"
",\n a.attnotnull");
@@ -1831,6 +1833,11 @@ describeOneTableDetails(const char *schemaname,
else
appendPQExpBufferStr(&buf, ",\n ''::pg_catalog.char AS attidentity");
attidentity_col = cols++;
+ if (pset.sversion >= 120000)
+ appendPQExpBufferStr(&buf, ",\n a.attgenerated");
+ else
+ appendPQExpBufferStr(&buf, ",\n ''::pg_catalog.char AS attgenerated");
+ attgenerated_col = cols++;
}
if (tableinfo.relkind == RELKIND_INDEX ||
tableinfo.relkind == RELKIND_PARTITIONED_INDEX)
@@ -2011,6 +2018,7 @@ describeOneTableDetails(const char *schemaname,
if (show_column_details)
{
char *identity;
+ char *generated;
char *default_str = "";
printTableAddCell(&cont, PQgetvalue(res, i, attcoll_col), false, false);
@@ -2020,16 +2028,19 @@ describeOneTableDetails(const char *schemaname,
false, false);
identity = PQgetvalue(res, i, attidentity_col);
+ generated = PQgetvalue(res, i, attgenerated_col);
- if (!identity[0])
- /* (note: above we cut off the 'default' string at 128) */
- default_str = PQgetvalue(res, i, attrdef_col);
- else if (identity[0] == ATTRIBUTE_IDENTITY_ALWAYS)
+ if (identity[0] == ATTRIBUTE_IDENTITY_ALWAYS)
default_str = "generated always as identity";
else if (identity[0] == ATTRIBUTE_IDENTITY_BY_DEFAULT)
default_str = "generated by default as identity";
+ else if (generated[0] == ATTRIBUTE_GENERATED_STORED)
+ default_str = psprintf("generated always as (%s) stored", PQgetvalue(res, i, attrdef_col));
+ else
+ /* (note: above we cut off the 'default' string at 128) */
+ default_str = PQgetvalue(res, i, attrdef_col);
- printTableAddCell(&cont, default_str, false, false);
+ printTableAddCell(&cont, default_str, false, generated[0] ? true : false);
}
/* Info for index columns */
diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h
index 66d1b2fc40..a592d22a0e 100644
--- a/src/include/access/tupdesc.h
+++ b/src/include/access/tupdesc.h
@@ -42,6 +42,7 @@ typedef struct TupleConstr
uint16 num_defval;
uint16 num_check;
bool has_not_null;
+ bool has_generated_stored;
} TupleConstr;
/*
diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h
index 85076d0743..83ff373c30 100644
--- a/src/include/catalog/heap.h
+++ b/src/include/catalog/heap.h
@@ -28,6 +28,7 @@ typedef struct RawColumnDefault
AttrNumber attnum; /* attribute to attach default to */
Node *raw_default; /* default value (untransformed parse tree) */
bool missingMode; /* true if part of add column processing */
+ char generated; /* attgenerated setting */
} RawColumnDefault;
typedef struct CookedConstraint
@@ -120,7 +121,8 @@ extern Node *cookDefault(ParseState *pstate,
Node *raw_default,
Oid atttypid,
int32 atttypmod,
- const char *attname);
+ const char *attname,
+ char attgenerated);
extern void DeleteRelationTuple(Oid relid);
extern void DeleteAttributeTuples(Oid relid);
diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h
index a6ec122389..04004b5703 100644
--- a/src/include/catalog/pg_attribute.h
+++ b/src/include/catalog/pg_attribute.h
@@ -140,6 +140,9 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75,
/* One of the ATTRIBUTE_IDENTITY_* constants below, or '\0' */
char attidentity BKI_DEFAULT('\0');
+ /* One of the ATTRIBUTE_GENERATED_* constants below, or '\0' */
+ char attgenerated BKI_DEFAULT('\0');
+
/* Is dropped (ie, logically invisible) or not */
bool attisdropped BKI_DEFAULT(f);
@@ -201,6 +204,8 @@ typedef FormData_pg_attribute *Form_pg_attribute;
#define ATTRIBUTE_IDENTITY_ALWAYS 'a'
#define ATTRIBUTE_IDENTITY_BY_DEFAULT 'd'
+#define ATTRIBUTE_GENERATED_STORED 's'
+
#endif /* EXPOSE_TO_CLIENT_CODE */
#endif /* PG_ATTRIBUTE_H */
diff --git a/src/include/catalog/pg_class.dat b/src/include/catalog/pg_class.dat
index c89710bc60..9bcf28676d 100644
--- a/src/include/catalog/pg_class.dat
+++ b/src/include/catalog/pg_class.dat
@@ -34,7 +34,7 @@
relname => 'pg_attribute', reltype => 'pg_attribute', relam => 'heap',
relfilenode => '0', relpages => '0', reltuples => '0', relallvisible => '0',
reltoastrelid => '0', relhasindex => 'f', relisshared => 'f',
- relpersistence => 'p', relkind => 'r', relnatts => '24', relchecks => '0',
+ relpersistence => 'p', relkind => 'r', relnatts => '25', relchecks => '0',
relhasrules => 'f', relhastriggers => 'f', relhassubclass => 'f',
relrowsecurity => 'f', relforcerowsecurity => 'f', relispopulated => 't',
relreplident => 'n', relispartition => 'f', relfrozenxid => '3',
diff --git a/src/include/executor/nodeModifyTable.h b/src/include/executor/nodeModifyTable.h
index b8b289efc0..891b119608 100644
--- a/src/include/executor/nodeModifyTable.h
+++ b/src/include/executor/nodeModifyTable.h
@@ -15,6 +15,8 @@
#include "nodes/execnodes.h"
+extern void ExecComputeStoredGenerated(EState *estate, TupleTableSlot *slot);
+
extern ModifyTableState *ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags);
extern void ExecEndModifyTable(ModifyTableState *node);
extern void ExecReScanModifyTable(ModifyTableState *node);
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 869c303e15..dbd7ed0363 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -452,6 +452,9 @@ typedef struct ResultRelInfo
/* array of constraint-checking expr states */
ExprState **ri_ConstraintExprs;
+ /* array of stored generated columns expr states */
+ ExprState **ri_GeneratedExprs;
+
/* for removing junk attributes from tuples */
JunkFilter *ri_junkFilter;
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index bdd2bd2fd9..4469e922ba 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -655,6 +655,7 @@ typedef struct ColumnDef
char identity; /* attidentity setting */
RangeVar *identitySequence; /* to store identity sequence name for
* ALTER TABLE ... ADD COLUMN */
+ char generated; /* attgenerated setting */
CollateClause *collClause; /* untransformed COLLATE spec, if any */
Oid collOid; /* collation OID (InvalidOid if not set) */
List *constraints; /* other constraints on column */
@@ -677,10 +678,11 @@ typedef enum TableLikeOption
CREATE_TABLE_LIKE_COMMENTS = 1 << 0,
CREATE_TABLE_LIKE_CONSTRAINTS = 1 << 1,
CREATE_TABLE_LIKE_DEFAULTS = 1 << 2,
- CREATE_TABLE_LIKE_IDENTITY = 1 << 3,
- CREATE_TABLE_LIKE_INDEXES = 1 << 4,
- CREATE_TABLE_LIKE_STATISTICS = 1 << 5,
- CREATE_TABLE_LIKE_STORAGE = 1 << 6,
+ CREATE_TABLE_LIKE_GENERATED = 1 << 3,
+ CREATE_TABLE_LIKE_IDENTITY = 1 << 4,
+ CREATE_TABLE_LIKE_INDEXES = 1 << 5,
+ CREATE_TABLE_LIKE_STATISTICS = 1 << 6,
+ CREATE_TABLE_LIKE_STORAGE = 1 << 7,
CREATE_TABLE_LIKE_ALL = PG_INT32_MAX
} TableLikeOption;
@@ -933,6 +935,15 @@ typedef struct PartitionCmd
* them in these fields. A whole-row Var reference is represented by
* setting the bit for InvalidAttrNumber.
*
+ * updatedCols is also used in some other places, for example, to determine
+ * which triggers to fire and in FDWs to know which changed columns they
+ * need to ship off. Generated columns that are caused to be updated by an
+ * update to a base column are collected in extraUpdatedCols. This is not
+ * considered for permission checking, but it is useful in those places
+ * that want to know the full set of columns being updated as opposed to
+ * only the ones the user explicitly mentioned in the query. (There is
+ * currently no need for an extraInsertedCols, but it could exist.)
+ *
* securityQuals is a list of security barrier quals (boolean expressions),
* to be tested in the listed order before returning a row from the
* relation. It is always NIL in parser output. Entries are added by the
@@ -1087,6 +1098,7 @@ typedef struct RangeTblEntry
Bitmapset *selectedCols; /* columns needing SELECT permission */
Bitmapset *insertedCols; /* columns needing INSERT permission */
Bitmapset *updatedCols; /* columns needing UPDATE permission */
+ Bitmapset *extraUpdatedCols; /* generated columns being updated */
List *securityQuals; /* security barrier quals to apply, if any */
} RangeTblEntry;
@@ -2086,6 +2098,7 @@ typedef enum ConstrType /* types of constraints */
CONSTR_NOTNULL,
CONSTR_DEFAULT,
CONSTR_IDENTITY,
+ CONSTR_GENERATED,
CONSTR_CHECK,
CONSTR_PRIMARY,
CONSTR_UNIQUE,
@@ -2124,7 +2137,8 @@ typedef struct Constraint
bool is_no_inherit; /* is constraint non-inheritable? */
Node *raw_expr; /* expr, as untransformed parse tree */
char *cooked_expr; /* expr, as nodeToString representation */
- char generated_when;
+ char generated_when; /* ALWAYS or BY DEFAULT */
+ char generated_kind; /* currently always STORED */
/* Fields used for unique constraints (UNIQUE and PRIMARY KEY): */
List *keys; /* String nodes naming referenced key
diff --git a/src/include/optimizer/plancat.h b/src/include/optimizer/plancat.h
index c337f047cb..c556e0f258 100644
--- a/src/include/optimizer/plancat.h
+++ b/src/include/optimizer/plancat.h
@@ -71,4 +71,6 @@ extern double get_function_rows(PlannerInfo *root, Oid funcid, Node *node);
extern bool has_row_triggers(PlannerInfo *root, Index rti, CmdType event);
+extern bool has_stored_generated_columns(PlannerInfo *root, Index rti);
+
#endif /* PLANCAT_H */
diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h
index f05444008c..00ace8425e 100644
--- a/src/include/parser/kwlist.h
+++ b/src/include/parser/kwlist.h
@@ -383,6 +383,7 @@ PG_KEYWORD("statistics", STATISTICS, UNRESERVED_KEYWORD)
PG_KEYWORD("stdin", STDIN, UNRESERVED_KEYWORD)
PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD)
PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD)
+PG_KEYWORD("stored", STORED, UNRESERVED_KEYWORD)
PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD)
PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD)
PG_KEYWORD("subscription", SUBSCRIPTION, UNRESERVED_KEYWORD)
diff --git a/src/include/parser/parse_node.h b/src/include/parser/parse_node.h
index ea99a0954b..3d8039aa51 100644
--- a/src/include/parser/parse_node.h
+++ b/src/include/parser/parse_node.h
@@ -71,7 +71,8 @@ typedef enum ParseExprKind
EXPR_KIND_PARTITION_BOUND, /* partition bound expression */
EXPR_KIND_PARTITION_EXPRESSION, /* PARTITION BY expression */
EXPR_KIND_CALL_ARGUMENT, /* procedure argument in CALL */
- EXPR_KIND_COPY_WHERE /* WHERE condition in COPY FROM */
+ EXPR_KIND_COPY_WHERE, /* WHERE condition in COPY FROM */
+ EXPR_KIND_GENERATED_COLUMN, /* generation expression for a column */
} ParseExprKind;
diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h
index b9a9ecb7cc..9606d021b1 100644
--- a/src/include/utils/lsyscache.h
+++ b/src/include/utils/lsyscache.h
@@ -86,6 +86,7 @@ extern Oid get_opfamily_proc(Oid opfamily, Oid lefttype, Oid righttype,
int16 procnum);
extern char *get_attname(Oid relid, AttrNumber attnum, bool missing_ok);
extern AttrNumber get_attnum(Oid relid, const char *attname);
+extern char get_attgenerated(Oid relid, AttrNumber attnum);
extern Oid get_atttype(Oid relid, AttrNumber attnum);
extern void get_atttypetypmodcoll(Oid relid, AttrNumber attnum,
Oid *typid, int32 *typmod, Oid *collid);
diff --git a/src/pl/plperl/expected/plperl_trigger.out b/src/pl/plperl/expected/plperl_trigger.out
index 28011cd9f6..d4879e2f03 100644
--- a/src/pl/plperl/expected/plperl_trigger.out
+++ b/src/pl/plperl/expected/plperl_trigger.out
@@ -6,6 +6,10 @@ CREATE TABLE trigger_test (
v varchar,
foo rowcompnest
);
+CREATE TABLE trigger_test_generated (
+ i int,
+ j int GENERATED ALWAYS AS (i * 2) STORED
+);
CREATE OR REPLACE FUNCTION trigger_data() RETURNS trigger LANGUAGE plperl AS $$
# make sure keys are sorted for consistent results - perl no longer
@@ -98,6 +102,79 @@ NOTICE: $_TD->{table_name} = 'trigger_test'
NOTICE: $_TD->{table_schema} = 'public'
NOTICE: $_TD->{when} = 'BEFORE'
DROP TRIGGER show_trigger_data_trig on trigger_test;
+CREATE TRIGGER show_trigger_data_trig_before
+BEFORE INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+CREATE TRIGGER show_trigger_data_trig_after
+AFTER INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+insert into trigger_test_generated (i) values (1);
+NOTICE: $_TD->{argc} = '0'
+NOTICE: $_TD->{event} = 'INSERT'
+NOTICE: $_TD->{level} = 'ROW'
+NOTICE: $_TD->{name} = 'show_trigger_data_trig_before'
+NOTICE: $_TD->{new} = {'i' => '1'}
+NOTICE: $_TD->{relid} = 'bogus:12345'
+NOTICE: $_TD->{relname} = 'trigger_test_generated'
+NOTICE: $_TD->{table_name} = 'trigger_test_generated'
+NOTICE: $_TD->{table_schema} = 'public'
+NOTICE: $_TD->{when} = 'BEFORE'
+NOTICE: $_TD->{argc} = '0'
+NOTICE: $_TD->{event} = 'INSERT'
+NOTICE: $_TD->{level} = 'ROW'
+NOTICE: $_TD->{name} = 'show_trigger_data_trig_after'
+NOTICE: $_TD->{new} = {'i' => '1', 'j' => '2'}
+NOTICE: $_TD->{relid} = 'bogus:12345'
+NOTICE: $_TD->{relname} = 'trigger_test_generated'
+NOTICE: $_TD->{table_name} = 'trigger_test_generated'
+NOTICE: $_TD->{table_schema} = 'public'
+NOTICE: $_TD->{when} = 'AFTER'
+update trigger_test_generated set i = 11 where i = 1;
+NOTICE: $_TD->{argc} = '0'
+NOTICE: $_TD->{event} = 'UPDATE'
+NOTICE: $_TD->{level} = 'ROW'
+NOTICE: $_TD->{name} = 'show_trigger_data_trig_before'
+NOTICE: $_TD->{new} = {'i' => '11'}
+NOTICE: $_TD->{old} = {'i' => '1', 'j' => '2'}
+NOTICE: $_TD->{relid} = 'bogus:12345'
+NOTICE: $_TD->{relname} = 'trigger_test_generated'
+NOTICE: $_TD->{table_name} = 'trigger_test_generated'
+NOTICE: $_TD->{table_schema} = 'public'
+NOTICE: $_TD->{when} = 'BEFORE'
+NOTICE: $_TD->{argc} = '0'
+NOTICE: $_TD->{event} = 'UPDATE'
+NOTICE: $_TD->{level} = 'ROW'
+NOTICE: $_TD->{name} = 'show_trigger_data_trig_after'
+NOTICE: $_TD->{new} = {'i' => '11', 'j' => '22'}
+NOTICE: $_TD->{old} = {'i' => '1', 'j' => '2'}
+NOTICE: $_TD->{relid} = 'bogus:12345'
+NOTICE: $_TD->{relname} = 'trigger_test_generated'
+NOTICE: $_TD->{table_name} = 'trigger_test_generated'
+NOTICE: $_TD->{table_schema} = 'public'
+NOTICE: $_TD->{when} = 'AFTER'
+delete from trigger_test_generated;
+NOTICE: $_TD->{argc} = '0'
+NOTICE: $_TD->{event} = 'DELETE'
+NOTICE: $_TD->{level} = 'ROW'
+NOTICE: $_TD->{name} = 'show_trigger_data_trig_before'
+NOTICE: $_TD->{old} = {'i' => '11', 'j' => '22'}
+NOTICE: $_TD->{relid} = 'bogus:12345'
+NOTICE: $_TD->{relname} = 'trigger_test_generated'
+NOTICE: $_TD->{table_name} = 'trigger_test_generated'
+NOTICE: $_TD->{table_schema} = 'public'
+NOTICE: $_TD->{when} = 'BEFORE'
+NOTICE: $_TD->{argc} = '0'
+NOTICE: $_TD->{event} = 'DELETE'
+NOTICE: $_TD->{level} = 'ROW'
+NOTICE: $_TD->{name} = 'show_trigger_data_trig_after'
+NOTICE: $_TD->{old} = {'i' => '11', 'j' => '22'}
+NOTICE: $_TD->{relid} = 'bogus:12345'
+NOTICE: $_TD->{relname} = 'trigger_test_generated'
+NOTICE: $_TD->{table_name} = 'trigger_test_generated'
+NOTICE: $_TD->{table_schema} = 'public'
+NOTICE: $_TD->{when} = 'AFTER'
+DROP TRIGGER show_trigger_data_trig_before ON trigger_test_generated;
+DROP TRIGGER show_trigger_data_trig_after ON trigger_test_generated;
insert into trigger_test values(1,'insert', '("(1)")');
CREATE VIEW trigger_test_view AS SELECT * FROM trigger_test;
CREATE TRIGGER show_trigger_data_trig
@@ -295,3 +372,21 @@ NOTICE: perlsnitch: ddl_command_start DROP TABLE
NOTICE: perlsnitch: ddl_command_end DROP TABLE
drop event trigger perl_a_snitch;
drop event trigger perl_b_snitch;
+-- dealing with generated columns
+CREATE FUNCTION generated_test_func1() RETURNS trigger
+LANGUAGE plperl
+AS $$
+$_TD->{new}{j} = 5; # not allowed
+return 'MODIFY';
+$$;
+CREATE TRIGGER generated_test_trigger1 BEFORE INSERT ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE generated_test_func1();
+TRUNCATE trigger_test_generated;
+INSERT INTO trigger_test_generated (i) VALUES (1);
+ERROR: cannot set generated column "j"
+CONTEXT: PL/Perl function "generated_test_func1"
+SELECT * FROM trigger_test_generated;
+ i | j
+---+---
+(0 rows)
+
diff --git a/src/pl/plperl/plperl.c b/src/pl/plperl/plperl.c
index 35d5d121a0..31ba2f262f 100644
--- a/src/pl/plperl/plperl.c
+++ b/src/pl/plperl/plperl.c
@@ -266,7 +266,7 @@ static plperl_proc_desc *compile_plperl_function(Oid fn_oid,
bool is_trigger,
bool is_event_trigger);
-static SV *plperl_hash_from_tuple(HeapTuple tuple, TupleDesc tupdesc);
+static SV *plperl_hash_from_tuple(HeapTuple tuple, TupleDesc tupdesc, bool include_generated);
static SV *plperl_hash_from_datum(Datum attr);
static SV *plperl_ref_from_pg_array(Datum arg, Oid typid);
static SV *split_array(plperl_array_info *info, int first, int last, int nest);
@@ -1644,13 +1644,19 @@ plperl_trigger_build_args(FunctionCallInfo fcinfo)
hv_store_string(hv, "name", cstr2sv(tdata->tg_trigger->tgname));
hv_store_string(hv, "relid", cstr2sv(relid));
+ /*
+ * Note: In BEFORE trigger, stored generated columns are not computed yet,
+ * so don't make them accessible in NEW row.
+ */
+
if (TRIGGER_FIRED_BY_INSERT(tdata->tg_event))
{
event = "INSERT";
if (TRIGGER_FIRED_FOR_ROW(tdata->tg_event))
hv_store_string(hv, "new",
plperl_hash_from_tuple(tdata->tg_trigtuple,
- tupdesc));
+ tupdesc,
+ !TRIGGER_FIRED_BEFORE(tdata->tg_event)));
}
else if (TRIGGER_FIRED_BY_DELETE(tdata->tg_event))
{
@@ -1658,7 +1664,8 @@ plperl_trigger_build_args(FunctionCallInfo fcinfo)
if (TRIGGER_FIRED_FOR_ROW(tdata->tg_event))
hv_store_string(hv, "old",
plperl_hash_from_tuple(tdata->tg_trigtuple,
- tupdesc));
+ tupdesc,
+ true));
}
else if (TRIGGER_FIRED_BY_UPDATE(tdata->tg_event))
{
@@ -1667,10 +1674,12 @@ plperl_trigger_build_args(FunctionCallInfo fcinfo)
{
hv_store_string(hv, "old",
plperl_hash_from_tuple(tdata->tg_trigtuple,
- tupdesc));
+ tupdesc,
+ true));
hv_store_string(hv, "new",
plperl_hash_from_tuple(tdata->tg_newtuple,
- tupdesc));
+ tupdesc,
+ !TRIGGER_FIRED_BEFORE(tdata->tg_event)));
}
}
else if (TRIGGER_FIRED_BY_TRUNCATE(tdata->tg_event))
@@ -1791,6 +1800,11 @@ plperl_modify_tuple(HV *hvTD, TriggerData *tdata, HeapTuple otup)
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("cannot set system attribute \"%s\"",
key)));
+ if (attr->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
+ errmsg("cannot set generated column \"%s\"",
+ key)));
modvalues[attn - 1] = plperl_sv_to_datum(val,
attr->atttypid,
@@ -3012,7 +3026,7 @@ plperl_hash_from_datum(Datum attr)
tmptup.t_len = HeapTupleHeaderGetDatumLength(td);
tmptup.t_data = td;
- sv = plperl_hash_from_tuple(&tmptup, tupdesc);
+ sv = plperl_hash_from_tuple(&tmptup, tupdesc, true);
ReleaseTupleDesc(tupdesc);
return sv;
@@ -3020,7 +3034,7 @@ plperl_hash_from_datum(Datum attr)
/* Build a hash from all attributes of a given tuple. */
static SV *
-plperl_hash_from_tuple(HeapTuple tuple, TupleDesc tupdesc)
+plperl_hash_from_tuple(HeapTuple tuple, TupleDesc tupdesc, bool include_generated)
{
dTHX;
HV *hv;
@@ -3044,6 +3058,13 @@ plperl_hash_from_tuple(HeapTuple tuple, TupleDesc tupdesc)
if (att->attisdropped)
continue;
+ if (att->attgenerated)
+ {
+ /* don't include unless requested */
+ if (!include_generated)
+ continue;
+ }
+
attname = NameStr(att->attname);
attr = heap_getattr(tuple, i + 1, tupdesc, &isnull);
@@ -3198,7 +3219,7 @@ plperl_spi_execute_fetch_result(SPITupleTable *tuptable, uint64 processed,
av_extend(rows, processed);
for (i = 0; i < processed; i++)
{
- row = plperl_hash_from_tuple(tuptable->vals[i], tuptable->tupdesc);
+ row = plperl_hash_from_tuple(tuptable->vals[i], tuptable->tupdesc, true);
av_push(rows, row);
}
hv_store_string(result, "rows",
@@ -3484,7 +3505,8 @@ plperl_spi_fetchrow(char *cursor)
else
{
row = plperl_hash_from_tuple(SPI_tuptable->vals[0],
- SPI_tuptable->tupdesc);
+ SPI_tuptable->tupdesc,
+ true);
}
SPI_freetuptable(SPI_tuptable);
}
diff --git a/src/pl/plperl/sql/plperl_trigger.sql b/src/pl/plperl/sql/plperl_trigger.sql
index 624193b9d0..4adddeb80a 100644
--- a/src/pl/plperl/sql/plperl_trigger.sql
+++ b/src/pl/plperl/sql/plperl_trigger.sql
@@ -8,6 +8,11 @@ CREATE TABLE trigger_test (
foo rowcompnest
);
+CREATE TABLE trigger_test_generated (
+ i int,
+ j int GENERATED ALWAYS AS (i * 2) STORED
+);
+
CREATE OR REPLACE FUNCTION trigger_data() RETURNS trigger LANGUAGE plperl AS $$
# make sure keys are sorted for consistent results - perl no longer
@@ -70,6 +75,21 @@ CREATE TRIGGER show_trigger_data_trig
DROP TRIGGER show_trigger_data_trig on trigger_test;
+CREATE TRIGGER show_trigger_data_trig_before
+BEFORE INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+
+CREATE TRIGGER show_trigger_data_trig_after
+AFTER INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+
+insert into trigger_test_generated (i) values (1);
+update trigger_test_generated set i = 11 where i = 1;
+delete from trigger_test_generated;
+
+DROP TRIGGER show_trigger_data_trig_before ON trigger_test_generated;
+DROP TRIGGER show_trigger_data_trig_after ON trigger_test_generated;
+
insert into trigger_test values(1,'insert', '("(1)")');
CREATE VIEW trigger_test_view AS SELECT * FROM trigger_test;
@@ -221,3 +241,19 @@ CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test
drop event trigger perl_a_snitch;
drop event trigger perl_b_snitch;
+
+-- dealing with generated columns
+
+CREATE FUNCTION generated_test_func1() RETURNS trigger
+LANGUAGE plperl
+AS $$
+$_TD->{new}{j} = 5; # not allowed
+return 'MODIFY';
+$$;
+
+CREATE TRIGGER generated_test_trigger1 BEFORE INSERT ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE generated_test_func1();
+
+TRUNCATE trigger_test_generated;
+INSERT INTO trigger_test_generated (i) VALUES (1);
+SELECT * FROM trigger_test_generated;
diff --git a/src/pl/plpgsql/src/pl_exec.c b/src/pl/plpgsql/src/pl_exec.c
index 527cada4fe..f0005009b2 100644
--- a/src/pl/plpgsql/src/pl_exec.c
+++ b/src/pl/plpgsql/src/pl_exec.c
@@ -924,6 +924,26 @@ plpgsql_exec_trigger(PLpgSQL_function *func,
false, false);
expanded_record_set_tuple(rec_old->erh, trigdata->tg_trigtuple,
false, false);
+
+ /*
+ * In BEFORE trigger, stored generated columns are not computed yet,
+ * so make them null in the NEW row. (Only needed in UPDATE branch;
+ * in the INSERT case, they are already null, but in UPDATE, the field
+ * still contains the old value.) Alternatively, we could construct a
+ * whole new row structure without the generated columns, but this way
+ * seems more efficient and potentially less confusing.
+ */
+ if (tupdesc->constr && tupdesc->constr->has_generated_stored &&
+ TRIGGER_FIRED_BEFORE(trigdata->tg_event))
+ {
+ for (int i = 0; i < tupdesc->natts; i++)
+ if (TupleDescAttr(tupdesc, i)->attgenerated == ATTRIBUTE_GENERATED_STORED)
+ expanded_record_set_field_internal(rec_new->erh,
+ i + 1,
+ (Datum) 0,
+ true, /*isnull*/
+ false, false);
+ }
}
else if (TRIGGER_FIRED_BY_DELETE(trigdata->tg_event))
{
diff --git a/src/pl/plpython/expected/plpython_trigger.out b/src/pl/plpython/expected/plpython_trigger.out
index d7ab8ac6b8..742988a5b5 100644
--- a/src/pl/plpython/expected/plpython_trigger.out
+++ b/src/pl/plpython/expected/plpython_trigger.out
@@ -67,6 +67,10 @@ SELECT * FROM users;
-- dump trigger data
CREATE TABLE trigger_test
(i int, v text );
+CREATE TABLE trigger_test_generated (
+ i int,
+ j int GENERATED ALWAYS AS (i * 2) STORED
+);
CREATE FUNCTION trigger_data() RETURNS trigger LANGUAGE plpythonu AS $$
if 'relid' in TD:
@@ -203,6 +207,77 @@ NOTICE: TD[when] => BEFORE
DROP TRIGGER show_trigger_data_trig_stmt on trigger_test;
DROP TRIGGER show_trigger_data_trig_before on trigger_test;
DROP TRIGGER show_trigger_data_trig_after on trigger_test;
+CREATE TRIGGER show_trigger_data_trig_before
+BEFORE INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+CREATE TRIGGER show_trigger_data_trig_after
+AFTER INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+insert into trigger_test_generated (i) values (1);
+NOTICE: TD[args] => None
+NOTICE: TD[event] => INSERT
+NOTICE: TD[level] => ROW
+NOTICE: TD[name] => show_trigger_data_trig_before
+NOTICE: TD[new] => {'i': 1}
+NOTICE: TD[old] => None
+NOTICE: TD[relid] => bogus:12345
+NOTICE: TD[table_name] => trigger_test_generated
+NOTICE: TD[table_schema] => public
+NOTICE: TD[when] => BEFORE
+NOTICE: TD[args] => None
+NOTICE: TD[event] => INSERT
+NOTICE: TD[level] => ROW
+NOTICE: TD[name] => show_trigger_data_trig_after
+NOTICE: TD[new] => {'i': 1, 'j': 2}
+NOTICE: TD[old] => None
+NOTICE: TD[relid] => bogus:12345
+NOTICE: TD[table_name] => trigger_test_generated
+NOTICE: TD[table_schema] => public
+NOTICE: TD[when] => AFTER
+update trigger_test_generated set i = 11 where i = 1;
+NOTICE: TD[args] => None
+NOTICE: TD[event] => UPDATE
+NOTICE: TD[level] => ROW
+NOTICE: TD[name] => show_trigger_data_trig_before
+NOTICE: TD[new] => {'i': 11}
+NOTICE: TD[old] => {'i': 1, 'j': 2}
+NOTICE: TD[relid] => bogus:12345
+NOTICE: TD[table_name] => trigger_test_generated
+NOTICE: TD[table_schema] => public
+NOTICE: TD[when] => BEFORE
+NOTICE: TD[args] => None
+NOTICE: TD[event] => UPDATE
+NOTICE: TD[level] => ROW
+NOTICE: TD[name] => show_trigger_data_trig_after
+NOTICE: TD[new] => {'i': 11, 'j': 22}
+NOTICE: TD[old] => {'i': 1, 'j': 2}
+NOTICE: TD[relid] => bogus:12345
+NOTICE: TD[table_name] => trigger_test_generated
+NOTICE: TD[table_schema] => public
+NOTICE: TD[when] => AFTER
+delete from trigger_test_generated;
+NOTICE: TD[args] => None
+NOTICE: TD[event] => DELETE
+NOTICE: TD[level] => ROW
+NOTICE: TD[name] => show_trigger_data_trig_before
+NOTICE: TD[new] => None
+NOTICE: TD[old] => {'i': 11, 'j': 22}
+NOTICE: TD[relid] => bogus:12345
+NOTICE: TD[table_name] => trigger_test_generated
+NOTICE: TD[table_schema] => public
+NOTICE: TD[when] => BEFORE
+NOTICE: TD[args] => None
+NOTICE: TD[event] => DELETE
+NOTICE: TD[level] => ROW
+NOTICE: TD[name] => show_trigger_data_trig_after
+NOTICE: TD[new] => None
+NOTICE: TD[old] => {'i': 11, 'j': 22}
+NOTICE: TD[relid] => bogus:12345
+NOTICE: TD[table_name] => trigger_test_generated
+NOTICE: TD[table_schema] => public
+NOTICE: TD[when] => AFTER
+DROP TRIGGER show_trigger_data_trig_before ON trigger_test_generated;
+DROP TRIGGER show_trigger_data_trig_after ON trigger_test_generated;
insert into trigger_test values(1,'insert');
CREATE VIEW trigger_test_view AS SELECT * FROM trigger_test;
CREATE TRIGGER show_trigger_data_trig
@@ -524,3 +599,22 @@ INFO: old: 1 -> a
INFO: new: 1 -> b
DROP TABLE transition_table_test;
DROP FUNCTION transition_table_test_f();
+-- dealing with generated columns
+CREATE FUNCTION generated_test_func1() RETURNS trigger
+LANGUAGE plpythonu
+AS $$
+TD['new']['j'] = 5 # not allowed
+return 'MODIFY'
+$$;
+CREATE TRIGGER generated_test_trigger1 BEFORE INSERT ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE generated_test_func1();
+TRUNCATE trigger_test_generated;
+INSERT INTO trigger_test_generated (i) VALUES (1);
+ERROR: cannot set generated column "j"
+CONTEXT: while modifying trigger row
+PL/Python function "generated_test_func1"
+SELECT * FROM trigger_test_generated;
+ i | j
+---+---
+(0 rows)
+
diff --git a/src/pl/plpython/plpy_cursorobject.c b/src/pl/plpython/plpy_cursorobject.c
index 45ac25b2ae..e4d543a4d4 100644
--- a/src/pl/plpython/plpy_cursorobject.c
+++ b/src/pl/plpython/plpy_cursorobject.c
@@ -357,7 +357,7 @@ PLy_cursor_iternext(PyObject *self)
exec_ctx->curr_proc);
ret = PLy_input_from_tuple(&cursor->result, SPI_tuptable->vals[0],
- SPI_tuptable->tupdesc);
+ SPI_tuptable->tupdesc, true);
}
SPI_freetuptable(SPI_tuptable);
@@ -453,7 +453,8 @@ PLy_cursor_fetch(PyObject *self, PyObject *args)
{
PyObject *row = PLy_input_from_tuple(&cursor->result,
SPI_tuptable->vals[i],
- SPI_tuptable->tupdesc);
+ SPI_tuptable->tupdesc,
+ true);
PyList_SetItem(ret->rows, i, row);
}
diff --git a/src/pl/plpython/plpy_exec.c b/src/pl/plpython/plpy_exec.c
index 2137186241..fd6cdc4ce5 100644
--- a/src/pl/plpython/plpy_exec.c
+++ b/src/pl/plpython/plpy_exec.c
@@ -13,6 +13,7 @@
#include "executor/spi.h"
#include "funcapi.h"
#include "utils/builtins.h"
+#include "utils/lsyscache.h"
#include "utils/rel.h"
#include "utils/typcache.h"
@@ -751,6 +752,11 @@ PLy_trigger_build_args(FunctionCallInfo fcinfo, PLyProcedure *proc, HeapTuple *r
PyDict_SetItemString(pltdata, "level", pltlevel);
Py_DECREF(pltlevel);
+ /*
+ * Note: In BEFORE trigger, stored generated columns are not computed yet,
+ * so don't make them accessible in NEW row.
+ */
+
if (TRIGGER_FIRED_BY_INSERT(tdata->tg_event))
{
pltevent = PyString_FromString("INSERT");
@@ -758,7 +764,8 @@ PLy_trigger_build_args(FunctionCallInfo fcinfo, PLyProcedure *proc, HeapTuple *r
PyDict_SetItemString(pltdata, "old", Py_None);
pytnew = PLy_input_from_tuple(&proc->result_in,
tdata->tg_trigtuple,
- rel_descr);
+ rel_descr,
+ !TRIGGER_FIRED_BEFORE(tdata->tg_event));
PyDict_SetItemString(pltdata, "new", pytnew);
Py_DECREF(pytnew);
*rv = tdata->tg_trigtuple;
@@ -770,7 +777,8 @@ PLy_trigger_build_args(FunctionCallInfo fcinfo, PLyProcedure *proc, HeapTuple *r
PyDict_SetItemString(pltdata, "new", Py_None);
pytold = PLy_input_from_tuple(&proc->result_in,
tdata->tg_trigtuple,
- rel_descr);
+ rel_descr,
+ true);
PyDict_SetItemString(pltdata, "old", pytold);
Py_DECREF(pytold);
*rv = tdata->tg_trigtuple;
@@ -781,12 +789,14 @@ PLy_trigger_build_args(FunctionCallInfo fcinfo, PLyProcedure *proc, HeapTuple *r
pytnew = PLy_input_from_tuple(&proc->result_in,
tdata->tg_newtuple,
- rel_descr);
+ rel_descr,
+ !TRIGGER_FIRED_BEFORE(tdata->tg_event));
PyDict_SetItemString(pltdata, "new", pytnew);
Py_DECREF(pytnew);
pytold = PLy_input_from_tuple(&proc->result_in,
tdata->tg_trigtuple,
- rel_descr);
+ rel_descr,
+ true);
PyDict_SetItemString(pltdata, "old", pytold);
Py_DECREF(pytold);
*rv = tdata->tg_newtuple;
@@ -952,6 +962,11 @@ PLy_modify_tuple(PLyProcedure *proc, PyObject *pltd, TriggerData *tdata,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("cannot set system attribute \"%s\"",
plattstr)));
+ if (TupleDescAttr(tupdesc, attn - 1)->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
+ errmsg("cannot set generated column \"%s\"",
+ plattstr)));
plval = PyDict_GetItem(plntup, platt);
if (plval == NULL)
diff --git a/src/pl/plpython/plpy_spi.c b/src/pl/plpython/plpy_spi.c
index 41155fc81e..fb23a7b3a4 100644
--- a/src/pl/plpython/plpy_spi.c
+++ b/src/pl/plpython/plpy_spi.c
@@ -419,7 +419,8 @@ PLy_spi_execute_fetch_result(SPITupleTable *tuptable, uint64 rows, int status)
{
PyObject *row = PLy_input_from_tuple(&ininfo,
tuptable->vals[i],
- tuptable->tupdesc);
+ tuptable->tupdesc,
+ true);
PyList_SetItem(result->rows, i, row);
}
diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c
index d6a6a849c3..6365e461e9 100644
--- a/src/pl/plpython/plpy_typeio.c
+++ b/src/pl/plpython/plpy_typeio.c
@@ -41,7 +41,7 @@ static PyObject *PLyList_FromArray(PLyDatumToOb *arg, Datum d);
static PyObject *PLyList_FromArray_recurse(PLyDatumToOb *elm, int *dims, int ndim, int dim,
char **dataptr_p, bits8 **bitmap_p, int *bitmask_p);
static PyObject *PLyDict_FromComposite(PLyDatumToOb *arg, Datum d);
-static PyObject *PLyDict_FromTuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc);
+static PyObject *PLyDict_FromTuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc, bool include_generated);
/* conversion from Python objects to Datums */
static Datum PLyObject_ToBool(PLyObToDatum *arg, PyObject *plrv,
@@ -134,7 +134,7 @@ PLy_output_convert(PLyObToDatum *arg, PyObject *val, bool *isnull)
* but in practice all callers have the right tupdesc available.
*/
PyObject *
-PLy_input_from_tuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc)
+PLy_input_from_tuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc, bool include_generated)
{
PyObject *dict;
PLyExecutionContext *exec_ctx = PLy_current_execution_context();
@@ -148,7 +148,7 @@ PLy_input_from_tuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc)
oldcontext = MemoryContextSwitchTo(scratch_context);
- dict = PLyDict_FromTuple(arg, tuple, desc);
+ dict = PLyDict_FromTuple(arg, tuple, desc, include_generated);
MemoryContextSwitchTo(oldcontext);
@@ -804,7 +804,7 @@ PLyDict_FromComposite(PLyDatumToOb *arg, Datum d)
tmptup.t_len = HeapTupleHeaderGetDatumLength(td);
tmptup.t_data = td;
- dict = PLyDict_FromTuple(arg, &tmptup, tupdesc);
+ dict = PLyDict_FromTuple(arg, &tmptup, tupdesc, true);
ReleaseTupleDesc(tupdesc);
@@ -815,7 +815,7 @@ PLyDict_FromComposite(PLyDatumToOb *arg, Datum d)
* Transform a tuple into a Python dict object.
*/
static PyObject *
-PLyDict_FromTuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc)
+PLyDict_FromTuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc, bool include_generated)
{
PyObject *volatile dict;
@@ -842,6 +842,13 @@ PLyDict_FromTuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc)
if (attr->attisdropped)
continue;
+ if (attr->attgenerated)
+ {
+ /* don't include unless requested */
+ if (!include_generated)
+ continue;
+ }
+
key = NameStr(attr->attname);
vattr = heap_getattr(tuple, (i + 1), desc, &is_null);
diff --git a/src/pl/plpython/plpy_typeio.h b/src/pl/plpython/plpy_typeio.h
index 82bdfae548..f210178238 100644
--- a/src/pl/plpython/plpy_typeio.h
+++ b/src/pl/plpython/plpy_typeio.h
@@ -151,7 +151,7 @@ extern Datum PLy_output_convert(PLyObToDatum *arg, PyObject *val,
bool *isnull);
extern PyObject *PLy_input_from_tuple(PLyDatumToOb *arg, HeapTuple tuple,
- TupleDesc desc);
+ TupleDesc desc, bool include_generated);
extern void PLy_input_setup_func(PLyDatumToOb *arg, MemoryContext arg_mcxt,
Oid typeOid, int32 typmod,
diff --git a/src/pl/plpython/sql/plpython_trigger.sql b/src/pl/plpython/sql/plpython_trigger.sql
index 79c24b714b..19852dc585 100644
--- a/src/pl/plpython/sql/plpython_trigger.sql
+++ b/src/pl/plpython/sql/plpython_trigger.sql
@@ -67,6 +67,11 @@ CREATE TRIGGER users_delete_trig BEFORE DELETE ON users FOR EACH ROW
CREATE TABLE trigger_test
(i int, v text );
+CREATE TABLE trigger_test_generated (
+ i int,
+ j int GENERATED ALWAYS AS (i * 2) STORED
+);
+
CREATE FUNCTION trigger_data() RETURNS trigger LANGUAGE plpythonu AS $$
if 'relid' in TD:
@@ -109,6 +114,21 @@ CREATE TRIGGER show_trigger_data_trig_stmt
DROP TRIGGER show_trigger_data_trig_before on trigger_test;
DROP TRIGGER show_trigger_data_trig_after on trigger_test;
+CREATE TRIGGER show_trigger_data_trig_before
+BEFORE INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+
+CREATE TRIGGER show_trigger_data_trig_after
+AFTER INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+
+insert into trigger_test_generated (i) values (1);
+update trigger_test_generated set i = 11 where i = 1;
+delete from trigger_test_generated;
+
+DROP TRIGGER show_trigger_data_trig_before ON trigger_test_generated;
+DROP TRIGGER show_trigger_data_trig_after ON trigger_test_generated;
+
insert into trigger_test values(1,'insert');
CREATE VIEW trigger_test_view AS SELECT * FROM trigger_test;
@@ -430,3 +450,20 @@ CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test
DROP TABLE transition_table_test;
DROP FUNCTION transition_table_test_f();
+
+
+-- dealing with generated columns
+
+CREATE FUNCTION generated_test_func1() RETURNS trigger
+LANGUAGE plpythonu
+AS $$
+TD['new']['j'] = 5 # not allowed
+return 'MODIFY'
+$$;
+
+CREATE TRIGGER generated_test_trigger1 BEFORE INSERT ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE generated_test_func1();
+
+TRUNCATE trigger_test_generated;
+INSERT INTO trigger_test_generated (i) VALUES (1);
+SELECT * FROM trigger_test_generated;
diff --git a/src/pl/tcl/expected/pltcl_trigger.out b/src/pl/tcl/expected/pltcl_trigger.out
index 2d5daedc11..008ea19509 100644
--- a/src/pl/tcl/expected/pltcl_trigger.out
+++ b/src/pl/tcl/expected/pltcl_trigger.out
@@ -61,6 +61,10 @@ CREATE TABLE trigger_test (
);
-- Make certain dropped attributes are handled correctly
ALTER TABLE trigger_test DROP dropme;
+CREATE TABLE trigger_test_generated (
+ i int,
+ j int GENERATED ALWAYS AS (i * 2) STORED
+);
CREATE VIEW trigger_test_view AS SELECT i, v FROM trigger_test;
CREATE FUNCTION trigger_data() returns trigger language pltcl as $_$
if {$TG_table_name eq "trigger_test" && $TG_level eq "ROW" && $TG_op ne "DELETE"} {
@@ -112,6 +116,12 @@ FOR EACH ROW EXECUTE PROCEDURE trigger_data(23,'skidoo');
CREATE TRIGGER statement_trigger
BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON trigger_test
FOR EACH STATEMENT EXECUTE PROCEDURE trigger_data(42,'statement trigger');
+CREATE TRIGGER show_trigger_data_trig_before
+BEFORE INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+CREATE TRIGGER show_trigger_data_trig_after
+AFTER INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
CREATE TRIGGER show_trigger_data_view_trig
INSTEAD OF INSERT OR UPDATE OR DELETE ON trigger_test_view
FOR EACH ROW EXECUTE PROCEDURE trigger_data(24,'skidoo view');
@@ -631,6 +641,75 @@ NOTICE: TG_table_name: trigger_test
NOTICE: TG_table_schema: public
NOTICE: TG_when: BEFORE
NOTICE: args: {23 skidoo}
+insert into trigger_test_generated (i) values (1);
+NOTICE: NEW: {i: 1}
+NOTICE: OLD: {}
+NOTICE: TG_level: ROW
+NOTICE: TG_name: show_trigger_data_trig_before
+NOTICE: TG_op: INSERT
+NOTICE: TG_relatts: {{} i j}
+NOTICE: TG_relid: bogus:12345
+NOTICE: TG_table_name: trigger_test_generated
+NOTICE: TG_table_schema: public
+NOTICE: TG_when: BEFORE
+NOTICE: args: {}
+NOTICE: NEW: {i: 1, j: 2}
+NOTICE: OLD: {}
+NOTICE: TG_level: ROW
+NOTICE: TG_name: show_trigger_data_trig_after
+NOTICE: TG_op: INSERT
+NOTICE: TG_relatts: {{} i j}
+NOTICE: TG_relid: bogus:12345
+NOTICE: TG_table_name: trigger_test_generated
+NOTICE: TG_table_schema: public
+NOTICE: TG_when: AFTER
+NOTICE: args: {}
+update trigger_test_generated set i = 11 where i = 1;
+NOTICE: NEW: {i: 11}
+NOTICE: OLD: {i: 1, j: 2}
+NOTICE: TG_level: ROW
+NOTICE: TG_name: show_trigger_data_trig_before
+NOTICE: TG_op: UPDATE
+NOTICE: TG_relatts: {{} i j}
+NOTICE: TG_relid: bogus:12345
+NOTICE: TG_table_name: trigger_test_generated
+NOTICE: TG_table_schema: public
+NOTICE: TG_when: BEFORE
+NOTICE: args: {}
+NOTICE: NEW: {i: 11, j: 22}
+NOTICE: OLD: {i: 1, j: 2}
+NOTICE: TG_level: ROW
+NOTICE: TG_name: show_trigger_data_trig_after
+NOTICE: TG_op: UPDATE
+NOTICE: TG_relatts: {{} i j}
+NOTICE: TG_relid: bogus:12345
+NOTICE: TG_table_name: trigger_test_generated
+NOTICE: TG_table_schema: public
+NOTICE: TG_when: AFTER
+NOTICE: args: {}
+delete from trigger_test_generated;
+NOTICE: NEW: {}
+NOTICE: OLD: {i: 11, j: 22}
+NOTICE: TG_level: ROW
+NOTICE: TG_name: show_trigger_data_trig_before
+NOTICE: TG_op: DELETE
+NOTICE: TG_relatts: {{} i j}
+NOTICE: TG_relid: bogus:12345
+NOTICE: TG_table_name: trigger_test_generated
+NOTICE: TG_table_schema: public
+NOTICE: TG_when: BEFORE
+NOTICE: args: {}
+NOTICE: NEW: {}
+NOTICE: OLD: {i: 11, j: 22}
+NOTICE: TG_level: ROW
+NOTICE: TG_name: show_trigger_data_trig_after
+NOTICE: TG_op: DELETE
+NOTICE: TG_relatts: {{} i j}
+NOTICE: TG_relid: bogus:12345
+NOTICE: TG_table_name: trigger_test_generated
+NOTICE: TG_table_schema: public
+NOTICE: TG_when: AFTER
+NOTICE: args: {}
insert into trigger_test_view values(2,'insert');
NOTICE: NEW: {i: 2, v: insert}
NOTICE: OLD: {}
@@ -738,6 +817,8 @@ NOTICE: TG_table_name: trigger_test
NOTICE: TG_table_schema: public
NOTICE: TG_when: BEFORE
NOTICE: args: {42 {statement trigger}}
+DROP TRIGGER show_trigger_data_trig_before ON trigger_test_generated;
+DROP TRIGGER show_trigger_data_trig_after ON trigger_test_generated;
-- should error
insert into trigger_test(test_argisnull) values(true);
NOTICE: NEW: {}
@@ -787,3 +868,21 @@ INFO: old: 1 -> a
INFO: new: 1 -> b
drop table transition_table_test;
drop function transition_table_test_f();
+-- dealing with generated columns
+CREATE FUNCTION generated_test_func1() RETURNS trigger
+LANGUAGE pltcl
+AS $$
+# not allowed
+set NEW(j) 5
+return [array get NEW]
+$$;
+CREATE TRIGGER generated_test_trigger1 BEFORE INSERT ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE generated_test_func1();
+TRUNCATE trigger_test_generated;
+INSERT INTO trigger_test_generated (i) VALUES (1);
+ERROR: cannot set generated column "j"
+SELECT * FROM trigger_test_generated;
+ i | j
+---+---
+(0 rows)
+
diff --git a/src/pl/tcl/pltcl.c b/src/pl/tcl/pltcl.c
index 76c9afc339..1362ca51d1 100644
--- a/src/pl/tcl/pltcl.c
+++ b/src/pl/tcl/pltcl.c
@@ -324,7 +324,7 @@ static void pltcl_subtrans_abort(Tcl_Interp *interp,
static void pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname,
uint64 tupno, HeapTuple tuple, TupleDesc tupdesc);
-static Tcl_Obj *pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc);
+static Tcl_Obj *pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc, bool include_generated);
static HeapTuple pltcl_build_tuple_result(Tcl_Interp *interp,
Tcl_Obj **kvObjv, int kvObjc,
pltcl_call_state *call_state);
@@ -889,7 +889,7 @@ pltcl_func_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state,
tmptup.t_len = HeapTupleHeaderGetDatumLength(td);
tmptup.t_data = td;
- list_tmp = pltcl_build_tuple_argument(&tmptup, tupdesc);
+ list_tmp = pltcl_build_tuple_argument(&tmptup, tupdesc, true);
Tcl_ListObjAppendElement(NULL, tcl_cmd, list_tmp);
ReleaseTupleDesc(tupdesc);
@@ -1060,7 +1060,6 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state,
volatile HeapTuple rettup;
Tcl_Obj *tcl_cmd;
Tcl_Obj *tcl_trigtup;
- Tcl_Obj *tcl_newtup;
int tcl_rc;
int i;
const char *result;
@@ -1162,20 +1161,22 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state,
Tcl_ListObjAppendElement(NULL, tcl_cmd,
Tcl_NewStringObj("ROW", -1));
- /* Build the data list for the trigtuple */
- tcl_trigtup = pltcl_build_tuple_argument(trigdata->tg_trigtuple,
- tupdesc);
-
/*
* Now the command part of the event for TG_op and data for NEW
* and OLD
+ *
+ * Note: In BEFORE trigger, stored generated columns are not computed yet,
+ * so don't make them accessible in NEW row.
*/
if (TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
{
Tcl_ListObjAppendElement(NULL, tcl_cmd,
Tcl_NewStringObj("INSERT", -1));
- Tcl_ListObjAppendElement(NULL, tcl_cmd, tcl_trigtup);
+ Tcl_ListObjAppendElement(NULL, tcl_cmd,
+ pltcl_build_tuple_argument(trigdata->tg_trigtuple,
+ tupdesc,
+ !TRIGGER_FIRED_BEFORE(trigdata->tg_event)));
Tcl_ListObjAppendElement(NULL, tcl_cmd, Tcl_NewObj());
rettup = trigdata->tg_trigtuple;
@@ -1186,7 +1187,10 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state,
Tcl_NewStringObj("DELETE", -1));
Tcl_ListObjAppendElement(NULL, tcl_cmd, Tcl_NewObj());
- Tcl_ListObjAppendElement(NULL, tcl_cmd, tcl_trigtup);
+ Tcl_ListObjAppendElement(NULL, tcl_cmd,
+ pltcl_build_tuple_argument(trigdata->tg_trigtuple,
+ tupdesc,
+ true));
rettup = trigdata->tg_trigtuple;
}
@@ -1195,11 +1199,14 @@ pltcl_trigger_handler(PG_FUNCTION_ARGS, pltcl_call_state *call_state,
Tcl_ListObjAppendElement(NULL, tcl_cmd,
Tcl_NewStringObj("UPDATE", -1));
- tcl_newtup = pltcl_build_tuple_argument(trigdata->tg_newtuple,
- tupdesc);
-
- Tcl_ListObjAppendElement(NULL, tcl_cmd, tcl_newtup);
- Tcl_ListObjAppendElement(NULL, tcl_cmd, tcl_trigtup);
+ Tcl_ListObjAppendElement(NULL, tcl_cmd,
+ pltcl_build_tuple_argument(trigdata->tg_newtuple,
+ tupdesc,
+ !TRIGGER_FIRED_BEFORE(trigdata->tg_event)));
+ Tcl_ListObjAppendElement(NULL, tcl_cmd,
+ pltcl_build_tuple_argument(trigdata->tg_trigtuple,
+ tupdesc,
+ true));
rettup = trigdata->tg_newtuple;
}
@@ -3091,7 +3098,7 @@ pltcl_set_tuple_values(Tcl_Interp *interp, const char *arrayname,
* from all attributes of a given tuple
**********************************************************************/
static Tcl_Obj *
-pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc)
+pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc, bool include_generated)
{
Tcl_Obj *retobj = Tcl_NewObj();
int i;
@@ -3110,6 +3117,13 @@ pltcl_build_tuple_argument(HeapTuple tuple, TupleDesc tupdesc)
if (att->attisdropped)
continue;
+ if (att->attgenerated)
+ {
+ /* don't include unless requested */
+ if (!include_generated)
+ continue;
+ }
+
/************************************************************
* Get the attribute name
************************************************************/
@@ -3219,6 +3233,12 @@ pltcl_build_tuple_result(Tcl_Interp *interp, Tcl_Obj **kvObjv, int kvObjc,
errmsg("cannot set system attribute \"%s\"",
fieldName)));
+ if (TupleDescAttr(tupdesc, attn - 1)->attgenerated)
+ ereport(ERROR,
+ (errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
+ errmsg("cannot set generated column \"%s\"",
+ fieldName)));
+
values[attn - 1] = utf_u2e(Tcl_GetString(kvObjv[i + 1]));
}
diff --git a/src/pl/tcl/sql/pltcl_trigger.sql b/src/pl/tcl/sql/pltcl_trigger.sql
index 277d9a0413..2db75a333a 100644
--- a/src/pl/tcl/sql/pltcl_trigger.sql
+++ b/src/pl/tcl/sql/pltcl_trigger.sql
@@ -71,6 +71,11 @@ CREATE TABLE trigger_test (
-- Make certain dropped attributes are handled correctly
ALTER TABLE trigger_test DROP dropme;
+CREATE TABLE trigger_test_generated (
+ i int,
+ j int GENERATED ALWAYS AS (i * 2) STORED
+);
+
CREATE VIEW trigger_test_view AS SELECT i, v FROM trigger_test;
CREATE FUNCTION trigger_data() returns trigger language pltcl as $_$
@@ -125,6 +130,13 @@ CREATE TRIGGER statement_trigger
BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON trigger_test
FOR EACH STATEMENT EXECUTE PROCEDURE trigger_data(42,'statement trigger');
+CREATE TRIGGER show_trigger_data_trig_before
+BEFORE INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+CREATE TRIGGER show_trigger_data_trig_after
+AFTER INSERT OR UPDATE OR DELETE ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE trigger_data();
+
CREATE TRIGGER show_trigger_data_view_trig
INSTEAD OF INSERT OR UPDATE OR DELETE ON trigger_test_view
FOR EACH ROW EXECUTE PROCEDURE trigger_data(24,'skidoo view');
@@ -531,6 +543,10 @@ CREATE TRIGGER show_trigger_data_view_trig
-- show dump of trigger data
insert into trigger_test values(1,'insert');
+insert into trigger_test_generated (i) values (1);
+update trigger_test_generated set i = 11 where i = 1;
+delete from trigger_test_generated;
+
insert into trigger_test_view values(2,'insert');
update trigger_test_view set v = 'update' where i=1;
delete from trigger_test_view;
@@ -540,6 +556,9 @@ CREATE TRIGGER show_trigger_data_view_trig
delete from trigger_test;
truncate trigger_test;
+DROP TRIGGER show_trigger_data_trig_before ON trigger_test_generated;
+DROP TRIGGER show_trigger_data_trig_after ON trigger_test_generated;
+
-- should error
insert into trigger_test(test_argisnull) values(true);
@@ -565,3 +584,20 @@ CREATE TRIGGER a_t AFTER UPDATE ON transition_table_test
update transition_table_test set name = 'b';
drop table transition_table_test;
drop function transition_table_test_f();
+
+-- dealing with generated columns
+
+CREATE FUNCTION generated_test_func1() RETURNS trigger
+LANGUAGE pltcl
+AS $$
+# not allowed
+set NEW(j) 5
+return [array get NEW]
+$$;
+
+CREATE TRIGGER generated_test_trigger1 BEFORE INSERT ON trigger_test_generated
+FOR EACH ROW EXECUTE PROCEDURE generated_test_func1();
+
+TRUNCATE trigger_test_generated;
+INSERT INTO trigger_test_generated (i) VALUES (1);
+SELECT * FROM trigger_test_generated;
diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out
index b582211270..31db405175 100644
--- a/src/test/regress/expected/create_table_like.out
+++ b/src/test/regress/expected/create_table_like.out
@@ -113,6 +113,52 @@ SELECT * FROM test_like_id_3; -- identity was copied and applied
(1 row)
DROP TABLE test_like_id_1, test_like_id_2, test_like_id_3;
+CREATE TABLE test_like_gen_1 (a int, b int GENERATED ALWAYS AS (a * 2) STORED);
+\d test_like_gen_1
+ Table "public.test_like_gen_1"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ a | integer | | |
+ b | integer | | | generated always as (a * 2) stored
+
+INSERT INTO test_like_gen_1 (a) VALUES (1);
+SELECT * FROM test_like_gen_1;
+ a | b
+---+---
+ 1 | 2
+(1 row)
+
+CREATE TABLE test_like_gen_2 (LIKE test_like_gen_1);
+\d test_like_gen_2
+ Table "public.test_like_gen_2"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | |
+ b | integer | | |
+
+INSERT INTO test_like_gen_2 (a) VALUES (1);
+SELECT * FROM test_like_gen_2;
+ a | b
+---+---
+ 1 |
+(1 row)
+
+CREATE TABLE test_like_gen_3 (LIKE test_like_gen_1 INCLUDING GENERATED);
+\d test_like_gen_3
+ Table "public.test_like_gen_3"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ a | integer | | |
+ b | integer | | | generated always as (a * 2) stored
+
+INSERT INTO test_like_gen_3 (a) VALUES (1);
+SELECT * FROM test_like_gen_3;
+ a | b
+---+---
+ 1 | 2
+(1 row)
+
+DROP TABLE test_like_gen_1, test_like_gen_2, test_like_gen_3;
CREATE TABLE inhg (x text, LIKE inhx INCLUDING INDEXES, y text); /* copies indexes */
INSERT INTO inhg VALUES (5, 10);
INSERT INTO inhg VALUES (20, 10); -- should fail
diff --git a/src/test/regress/expected/generated.out b/src/test/regress/expected/generated.out
new file mode 100644
index 0000000000..e0d1d88f17
--- /dev/null
+++ b/src/test/regress/expected/generated.out
@@ -0,0 +1,768 @@
+-- sanity check of system catalog
+SELECT attrelid, attname, attgenerated FROM pg_attribute WHERE attgenerated NOT IN ('', 's');
+ attrelid | attname | attgenerated
+----------+---------+--------------
+(0 rows)
+
+CREATE TABLE gtest0 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (55) STORED);
+CREATE TABLE gtest1 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+SELECT table_name, column_name, column_default, is_nullable, is_generated, generation_expression FROM information_schema.columns WHERE table_name LIKE 'gtest_' ORDER BY 1, 2;
+ table_name | column_name | column_default | is_nullable | is_generated | generation_expression
+------------+-------------+----------------+-------------+--------------+-----------------------
+ gtest0 | a | | NO | NEVER |
+ gtest0 | b | | YES | ALWAYS | 55
+ gtest1 | a | | NO | NEVER |
+ gtest1 | b | | YES | ALWAYS | (a * 2)
+(4 rows)
+
+SELECT table_name, column_name, dependent_column FROM information_schema.column_column_usage ORDER BY 1, 2, 3;
+ table_name | column_name | dependent_column
+------------+-------------+------------------
+ gtest1 | a | b
+(1 row)
+
+\d gtest1
+ Table "public.gtest1"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ a | integer | | not null |
+ b | integer | | | generated always as (a * 2) stored
+Indexes:
+ "gtest1_pkey" PRIMARY KEY, btree (a)
+
+-- duplicate generated
+CREATE TABLE gtest_err_1 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED GENERATED ALWAYS AS (a * 3) STORED);
+ERROR: multiple generation clauses specified for column "b" of table "gtest_err_1"
+LINE 1: ...ARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED GENERATED ...
+ ^
+-- references to other generated columns, including self-references
+CREATE TABLE gtest_err_2a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (b * 2) STORED);
+ERROR: cannot use generated column "b" in column generation expression
+LINE 1: ...2a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (b * 2) STO...
+ ^
+DETAIL: A generated column cannot reference another generated column.
+CREATE TABLE gtest_err_2b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED, c int GENERATED ALWAYS AS (b * 3) STORED);
+ERROR: cannot use generated column "b" in column generation expression
+LINE 1: ...AYS AS (a * 2) STORED, c int GENERATED ALWAYS AS (b * 3) STO...
+ ^
+DETAIL: A generated column cannot reference another generated column.
+-- invalid reference
+CREATE TABLE gtest_err_3 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (c * 2) STORED);
+ERROR: column "c" does not exist
+LINE 1: ..._3 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (c * 2) STO...
+ ^
+-- generation expression must be immutable
+CREATE TABLE gtest_err_4 (a int PRIMARY KEY, b double precision GENERATED ALWAYS AS (random()) STORED);
+ERROR: generation expression is not immutable
+-- cannot have default/identity and generated
+CREATE TABLE gtest_err_5a (a int PRIMARY KEY, b int DEFAULT 5 GENERATED ALWAYS AS (a * 2) STORED);
+ERROR: both default and generation expression specified for column "b" of table "gtest_err_5a"
+LINE 1: ... gtest_err_5a (a int PRIMARY KEY, b int DEFAULT 5 GENERATED ...
+ ^
+CREATE TABLE gtest_err_5b (a int PRIMARY KEY, b int GENERATED ALWAYS AS identity GENERATED ALWAYS AS (a * 2) STORED);
+ERROR: both identity and generation expression specified for column "b" of table "gtest_err_5b"
+LINE 1: ...t PRIMARY KEY, b int GENERATED ALWAYS AS identity GENERATED ...
+ ^
+-- reference to system column not allowed in generated column
+CREATE TABLE gtest_err_6a (a int PRIMARY KEY, b bool GENERATED ALWAYS AS (xmin <> 37) STORED);
+ERROR: cannot use system column "xmin" in column generation expression
+LINE 1: ...a (a int PRIMARY KEY, b bool GENERATED ALWAYS AS (xmin <> 37...
+ ^
+-- various prohibited constructs
+CREATE TABLE gtest_err_7a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (avg(a)) STORED);
+ERROR: aggregate functions are not allowed in column generation expressions
+LINE 1: ...7a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (avg(a)) ST...
+ ^
+CREATE TABLE gtest_err_7b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (row_number() OVER (ORDER BY a)) STORED);
+ERROR: window functions are not allowed in column generation expressions
+LINE 1: ...7b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (row_number...
+ ^
+CREATE TABLE gtest_err_7c (a int PRIMARY KEY, b int GENERATED ALWAYS AS ((SELECT a)) STORED);
+ERROR: cannot use subquery in column generation expression
+LINE 1: ...7c (a int PRIMARY KEY, b int GENERATED ALWAYS AS ((SELECT a)...
+ ^
+CREATE TABLE gtest_err_7d (a int PRIMARY KEY, b int GENERATED ALWAYS AS (generate_series(1, a)) STORED);
+ERROR: set-returning functions are not allowed in column generation expressions
+LINE 1: ...7d (a int PRIMARY KEY, b int GENERATED ALWAYS AS (generate_s...
+ ^
+INSERT INTO gtest1 VALUES (1);
+INSERT INTO gtest1 VALUES (2, DEFAULT);
+INSERT INTO gtest1 VALUES (3, 33); -- error
+ERROR: cannot insert into column "b"
+DETAIL: Column "b" is a generated column.
+SELECT * FROM gtest1 ORDER BY a;
+ a | b
+---+---
+ 1 | 2
+ 2 | 4
+(2 rows)
+
+UPDATE gtest1 SET b = DEFAULT WHERE a = 1;
+UPDATE gtest1 SET b = 11 WHERE a = 1; -- error
+ERROR: column "b" can only be updated to DEFAULT
+DETAIL: Column "b" is a generated column.
+SELECT * FROM gtest1 ORDER BY a;
+ a | b
+---+---
+ 1 | 2
+ 2 | 4
+(2 rows)
+
+SELECT a, b, b * 2 AS b2 FROM gtest1 ORDER BY a;
+ a | b | b2
+---+---+----
+ 1 | 2 | 4
+ 2 | 4 | 8
+(2 rows)
+
+SELECT a, b FROM gtest1 WHERE b = 4 ORDER BY a;
+ a | b
+---+---
+ 2 | 4
+(1 row)
+
+-- test that overflow error happens on write
+INSERT INTO gtest1 VALUES (2000000000);
+ERROR: integer out of range
+SELECT * FROM gtest1;
+ a | b
+---+---
+ 2 | 4
+ 1 | 2
+(2 rows)
+
+DELETE FROM gtest1 WHERE a = 2000000000;
+-- test with joins
+CREATE TABLE gtestx (x int, y int);
+INSERT INTO gtestx VALUES (11, 1), (22, 2), (33, 3);
+SELECT * FROM gtestx, gtest1 WHERE gtestx.y = gtest1.a;
+ x | y | a | b
+----+---+---+---
+ 11 | 1 | 1 | 2
+ 22 | 2 | 2 | 4
+(2 rows)
+
+DROP TABLE gtestx;
+-- test UPDATE/DELETE quals
+SELECT * FROM gtest1 ORDER BY a;
+ a | b
+---+---
+ 1 | 2
+ 2 | 4
+(2 rows)
+
+UPDATE gtest1 SET a = 3 WHERE b = 4;
+SELECT * FROM gtest1 ORDER BY a;
+ a | b
+---+---
+ 1 | 2
+ 3 | 6
+(2 rows)
+
+DELETE FROM gtest1 WHERE b = 2;
+SELECT * FROM gtest1 ORDER BY a;
+ a | b
+---+---
+ 3 | 6
+(1 row)
+
+-- views
+CREATE VIEW gtest1v AS SELECT * FROM gtest1;
+SELECT * FROM gtest1v;
+ a | b
+---+---
+ 3 | 6
+(1 row)
+
+INSERT INTO gtest1v VALUES (4, 8); -- fails
+ERROR: cannot insert into column "b"
+DETAIL: Column "b" is a generated column.
+DROP VIEW gtest1v;
+-- CTEs
+WITH foo AS (SELECT * FROM gtest1) SELECT * FROM foo;
+ a | b
+---+---
+ 3 | 6
+(1 row)
+
+-- inheritance
+CREATE TABLE gtest1_1 () INHERITS (gtest1);
+SELECT * FROM gtest1_1;
+ a | b
+---+---
+(0 rows)
+
+\d gtest1_1
+ Table "public.gtest1_1"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ a | integer | | not null |
+ b | integer | | | generated always as (a * 2) stored
+Inherits: gtest1
+
+INSERT INTO gtest1_1 VALUES (4);
+SELECT * FROM gtest1_1;
+ a | b
+---+---
+ 4 | 8
+(1 row)
+
+SELECT * FROM gtest1;
+ a | b
+---+---
+ 3 | 6
+ 4 | 8
+(2 rows)
+
+-- test inheritance mismatch
+CREATE TABLE gtesty (x int, b int);
+CREATE TABLE gtest1_2 () INHERITS (gtest1, gtesty); -- error
+NOTICE: merging multiple inherited definitions of column "b"
+ERROR: inherited column "b" has a generation conflict
+DROP TABLE gtesty;
+-- test stored update
+CREATE TABLE gtest3 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 3) STORED);
+INSERT INTO gtest3 (a) VALUES (1), (2), (3);
+SELECT * FROM gtest3 ORDER BY a;
+ a | b
+---+---
+ 1 | 3
+ 2 | 6
+ 3 | 9
+(3 rows)
+
+UPDATE gtest3 SET a = 22 WHERE a = 2;
+SELECT * FROM gtest3 ORDER BY a;
+ a | b
+----+----
+ 1 | 3
+ 3 | 9
+ 22 | 66
+(3 rows)
+
+-- COPY
+TRUNCATE gtest1;
+INSERT INTO gtest1 (a) VALUES (1), (2);
+COPY gtest1 TO stdout;
+1
+2
+COPY gtest1 (a, b) TO stdout;
+ERROR: column "b" is a generated column
+DETAIL: Generated columns cannot be used in COPY.
+COPY gtest1 FROM stdin;
+COPY gtest1 (a, b) FROM stdin;
+ERROR: column "b" is a generated column
+DETAIL: Generated columns cannot be used in COPY.
+SELECT * FROM gtest1 ORDER BY a;
+ a | b
+---+---
+ 1 | 2
+ 2 | 4
+ 3 | 6
+ 4 | 8
+(4 rows)
+
+TRUNCATE gtest3;
+INSERT INTO gtest3 (a) VALUES (1), (2);
+COPY gtest3 TO stdout;
+1
+2
+COPY gtest3 (a, b) TO stdout;
+ERROR: column "b" is a generated column
+DETAIL: Generated columns cannot be used in COPY.
+COPY gtest3 FROM stdin;
+COPY gtest3 (a, b) FROM stdin;
+ERROR: column "b" is a generated column
+DETAIL: Generated columns cannot be used in COPY.
+SELECT * FROM gtest3 ORDER BY a;
+ a | b
+---+----
+ 1 | 3
+ 2 | 6
+ 3 | 9
+ 4 | 12
+(4 rows)
+
+-- null values
+CREATE TABLE gtest2 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (NULL) STORED);
+INSERT INTO gtest2 VALUES (1);
+SELECT * FROM gtest2;
+ a | b
+---+---
+ 1 |
+(1 row)
+
+-- composite types
+CREATE TYPE double_int as (a int, b int);
+CREATE TABLE gtest4 (
+ a int,
+ b double_int GENERATED ALWAYS AS ((a * 2, a * 3)) STORED
+);
+INSERT INTO gtest4 VALUES (1), (6);
+SELECT * FROM gtest4;
+ a | b
+---+---------
+ 1 | (2,3)
+ 6 | (12,18)
+(2 rows)
+
+DROP TABLE gtest4;
+DROP TYPE double_int;
+-- using tableoid is allowed
+CREATE TABLE gtest_tableoid (
+ a int PRIMARY KEY,
+ b bool GENERATED ALWAYS AS (tableoid <> 0) STORED
+);
+INSERT INTO gtest_tableoid VALUES (1), (2);
+SELECT * FROM gtest_tableoid;
+ a | b
+---+---
+ 1 | t
+ 2 | t
+(2 rows)
+
+-- drop column behavior
+CREATE TABLE gtest10 (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (b * 2) STORED);
+ALTER TABLE gtest10 DROP COLUMN b;
+\d gtest10
+ Table "public.gtest10"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+---------
+ a | integer | | not null |
+Indexes:
+ "gtest10_pkey" PRIMARY KEY, btree (a)
+
+CREATE TABLE gtest10a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+ALTER TABLE gtest10a DROP COLUMN b;
+INSERT INTO gtest10a (a) VALUES (1);
+-- privileges
+CREATE USER regress_user11;
+CREATE TABLE gtest11s (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (b * 2) STORED);
+INSERT INTO gtest11s VALUES (1, 10), (2, 20);
+GRANT SELECT (a, c) ON gtest11s TO regress_user11;
+CREATE FUNCTION gf1(a int) RETURNS int AS $$ SELECT a * 3 $$ IMMUTABLE LANGUAGE SQL;
+REVOKE ALL ON FUNCTION gf1(int) FROM PUBLIC;
+CREATE TABLE gtest12s (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (gf1(b)) STORED);
+INSERT INTO gtest12s VALUES (1, 10), (2, 20);
+GRANT SELECT (a, c) ON gtest12s TO regress_user11;
+SET ROLE regress_user11;
+SELECT a, b FROM gtest11s; -- not allowed
+ERROR: permission denied for table gtest11s
+SELECT a, c FROM gtest11s; -- allowed
+ a | c
+---+----
+ 1 | 20
+ 2 | 40
+(2 rows)
+
+SELECT gf1(10); -- not allowed
+ERROR: permission denied for function gf1
+SELECT a, c FROM gtest12s; -- allowed
+ a | c
+---+----
+ 1 | 30
+ 2 | 60
+(2 rows)
+
+RESET ROLE;
+DROP TABLE gtest11s, gtest12s;
+DROP FUNCTION gf1(int);
+DROP USER regress_user11;
+-- check constraints
+CREATE TABLE gtest20 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED CHECK (b < 50));
+INSERT INTO gtest20 (a) VALUES (10); -- ok
+INSERT INTO gtest20 (a) VALUES (30); -- violates constraint
+ERROR: new row for relation "gtest20" violates check constraint "gtest20_b_check"
+DETAIL: Failing row contains (30, 60).
+CREATE TABLE gtest20a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+INSERT INTO gtest20a (a) VALUES (10);
+INSERT INTO gtest20a (a) VALUES (30);
+ALTER TABLE gtest20a ADD CHECK (b < 50); -- fails on existing row
+ERROR: check constraint "gtest20a_b_check" is violated by some row
+CREATE TABLE gtest20b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+INSERT INTO gtest20b (a) VALUES (10);
+INSERT INTO gtest20b (a) VALUES (30);
+ALTER TABLE gtest20b ADD CONSTRAINT chk CHECK (b < 50) NOT VALID;
+ALTER TABLE gtest20b VALIDATE CONSTRAINT chk; -- fails on existing row
+ERROR: check constraint "chk" is violated by some row
+-- not-null constraints
+CREATE TABLE gtest21a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (nullif(a, 0)) STORED NOT NULL);
+INSERT INTO gtest21a (a) VALUES (1); -- ok
+INSERT INTO gtest21a (a) VALUES (0); -- violates constraint
+ERROR: null value in column "b" violates not-null constraint
+DETAIL: Failing row contains (0, null).
+CREATE TABLE gtest21b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (nullif(a, 0)) STORED);
+ALTER TABLE gtest21b ALTER COLUMN b SET NOT NULL;
+INSERT INTO gtest21b (a) VALUES (1); -- ok
+INSERT INTO gtest21b (a) VALUES (0); -- violates constraint
+ERROR: null value in column "b" violates not-null constraint
+DETAIL: Failing row contains (0, null).
+ALTER TABLE gtest21b ALTER COLUMN b DROP NOT NULL;
+INSERT INTO gtest21b (a) VALUES (0); -- ok now
+-- index constraints
+CREATE TABLE gtest22a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a / 2) STORED UNIQUE);
+INSERT INTO gtest22a VALUES (2);
+INSERT INTO gtest22a VALUES (3);
+ERROR: duplicate key value violates unique constraint "gtest22a_b_key"
+DETAIL: Key (b)=(1) already exists.
+INSERT INTO gtest22a VALUES (4);
+CREATE TABLE gtest22b (a int, b int GENERATED ALWAYS AS (a / 2) STORED, PRIMARY KEY (a, b));
+INSERT INTO gtest22b VALUES (2);
+INSERT INTO gtest22b VALUES (2);
+ERROR: duplicate key value violates unique constraint "gtest22b_pkey"
+DETAIL: Key (a, b)=(2, 1) already exists.
+-- indexes
+CREATE TABLE gtest22c (a int, b int GENERATED ALWAYS AS (a * 2) STORED);
+CREATE INDEX gtest22c_b_idx ON gtest22c (b);
+CREATE INDEX gtest22c_expr_idx ON gtest22c ((b * 3));
+CREATE INDEX gtest22c_pred_idx ON gtest22c (a) WHERE b > 0;
+\d gtest22c
+ Table "public.gtest22c"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ a | integer | | |
+ b | integer | | | generated always as (a * 2) stored
+Indexes:
+ "gtest22c_b_idx" btree (b)
+ "gtest22c_expr_idx" btree ((b * 3))
+ "gtest22c_pred_idx" btree (a) WHERE b > 0
+
+INSERT INTO gtest22c VALUES (1), (2), (3);
+SET enable_seqscan TO off;
+SET enable_bitmapscan TO off;
+EXPLAIN (COSTS OFF) SELECT * FROM gtest22c WHERE b = 4;
+ QUERY PLAN
+---------------------------------------------
+ Index Scan using gtest22c_b_idx on gtest22c
+ Index Cond: (b = 4)
+(2 rows)
+
+SELECT * FROM gtest22c WHERE b = 4;
+ a | b
+---+---
+ 2 | 4
+(1 row)
+
+EXPLAIN (COSTS OFF) SELECT * FROM gtest22c WHERE b * 3 = 6;
+ QUERY PLAN
+------------------------------------------------
+ Index Scan using gtest22c_expr_idx on gtest22c
+ Index Cond: ((b * 3) = 6)
+(2 rows)
+
+SELECT * FROM gtest22c WHERE b * 3 = 6;
+ a | b
+---+---
+ 1 | 2
+(1 row)
+
+EXPLAIN (COSTS OFF) SELECT * FROM gtest22c WHERE a = 1 AND b > 0;
+ QUERY PLAN
+------------------------------------------------
+ Index Scan using gtest22c_pred_idx on gtest22c
+ Index Cond: (a = 1)
+(2 rows)
+
+SELECT * FROM gtest22c WHERE a = 1 AND b > 0;
+ a | b
+---+---
+ 1 | 2
+(1 row)
+
+RESET enable_seqscan;
+RESET enable_bitmapscan;
+-- foreign keys
+CREATE TABLE gtest23a (x int PRIMARY KEY, y int);
+INSERT INTO gtest23a VALUES (1, 11), (2, 22), (3, 33);
+CREATE TABLE gtest23x (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED REFERENCES gtest23a (x) ON UPDATE CASCADE); -- error
+ERROR: invalid ON UPDATE action for foreign key constraint containing generated column
+CREATE TABLE gtest23x (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED REFERENCES gtest23a (x) ON DELETE SET NULL); -- error
+ERROR: invalid ON DELETE action for foreign key constraint containing generated column
+CREATE TABLE gtest23b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED REFERENCES gtest23a (x));
+\d gtest23b
+ Table "public.gtest23b"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ a | integer | | not null |
+ b | integer | | | generated always as (a * 2) stored
+Indexes:
+ "gtest23b_pkey" PRIMARY KEY, btree (a)
+Foreign-key constraints:
+ "gtest23b_b_fkey" FOREIGN KEY (b) REFERENCES gtest23a(x)
+
+INSERT INTO gtest23b VALUES (1); -- ok
+INSERT INTO gtest23b VALUES (5); -- error
+ERROR: insert or update on table "gtest23b" violates foreign key constraint "gtest23b_b_fkey"
+DETAIL: Key (b)=(10) is not present in table "gtest23a".
+DROP TABLE gtest23b;
+DROP TABLE gtest23a;
+CREATE TABLE gtest23p (x int, y int GENERATED ALWAYS AS (x * 2) STORED, PRIMARY KEY (y));
+INSERT INTO gtest23p VALUES (1), (2), (3);
+CREATE TABLE gtest23q (a int PRIMARY KEY, b int REFERENCES gtest23p (y));
+INSERT INTO gtest23q VALUES (1, 2); -- ok
+INSERT INTO gtest23q VALUES (2, 5); -- error
+ERROR: insert or update on table "gtest23q" violates foreign key constraint "gtest23q_b_fkey"
+DETAIL: Key (b)=(5) is not present in table "gtest23p".
+-- domains
+CREATE DOMAIN gtestdomain1 AS int CHECK (VALUE < 10);
+CREATE TABLE gtest24 (a int PRIMARY KEY, b gtestdomain1 GENERATED ALWAYS AS (a * 2) STORED);
+INSERT INTO gtest24 (a) VALUES (4); -- ok
+INSERT INTO gtest24 (a) VALUES (6); -- error
+ERROR: value for domain gtestdomain1 violates check constraint "gtestdomain1_check"
+-- typed tables (currently not supported)
+CREATE TYPE gtest_type AS (f1 integer, f2 text, f3 bigint);
+CREATE TABLE gtest28 OF gtest_type (f1 WITH OPTIONS GENERATED ALWAYS AS (f2 *2) STORED);
+ERROR: generated columns are not supported on typed tables
+DROP TYPE gtest_type CASCADE;
+-- table partitions (currently not supported)
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 text, f3 bigint) PARTITION BY RANGE (f1);
+CREATE TABLE gtest_child PARTITION OF gtest_parent (
+ f3 WITH OPTIONS GENERATED ALWAYS AS (f2 * 2) STORED
+) FOR VALUES FROM ('2016-07-01') TO ('2016-08-01'); -- error
+ERROR: generated columns are not supported on partitions
+DROP TABLE gtest_parent;
+-- partitioned table
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 bigint, f3 bigint GENERATED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE (f1);
+CREATE TABLE gtest_child PARTITION OF gtest_parent FOR VALUES FROM ('2016-07-01') TO ('2016-08-01');
+INSERT INTO gtest_parent (f1, f2) VALUES ('2016-07-15', 1);
+SELECT * FROM gtest_parent;
+ f1 | f2 | f3
+------------+----+----
+ 07-15-2016 | 1 | 2
+(1 row)
+
+SELECT * FROM gtest_child;
+ f1 | f2 | f3
+------------+----+----
+ 07-15-2016 | 1 | 2
+(1 row)
+
+DROP TABLE gtest_parent;
+-- generated columns in partition key (not allowed)
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 bigint, f3 bigint GENERATED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE (f3);
+ERROR: cannot use generated column in partition key
+LINE 1: ...ENERATED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE (f3);
+ ^
+DETAIL: Column "f3" is a generated column.
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 bigint, f3 bigint GENERATED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE ((f3 * 3));
+ERROR: cannot use generated column in partition key
+LINE 1: ...ED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE ((f3 * 3));
+ ^
+DETAIL: Column "f3" is a generated column.
+-- ALTER TABLE ... ADD COLUMN
+CREATE TABLE gtest25 (a int PRIMARY KEY);
+INSERT INTO gtest25 VALUES (3), (4);
+ALTER TABLE gtest25 ADD COLUMN b int GENERATED ALWAYS AS (a * 3) STORED;
+SELECT * FROM gtest25 ORDER BY a;
+ a | b
+---+----
+ 3 | 9
+ 4 | 12
+(2 rows)
+
+ALTER TABLE gtest25 ADD COLUMN x int GENERATED ALWAYS AS (b * 4) STORED; -- error
+ERROR: cannot use generated column "b" in column generation expression
+DETAIL: A generated column cannot reference another generated column.
+ALTER TABLE gtest25 ADD COLUMN x int GENERATED ALWAYS AS (z * 4) STORED; -- error
+ERROR: column "z" does not exist
+-- ALTER TABLE ... ALTER COLUMN
+CREATE TABLE gtest27 (
+ a int,
+ b int GENERATED ALWAYS AS (a * 2) STORED
+);
+INSERT INTO gtest27 (a) VALUES (3), (4);
+ALTER TABLE gtest27 ALTER COLUMN a TYPE text; -- error
+ERROR: cannot alter type of a column used by a generated column
+DETAIL: Column "a" is used by generated column "b".
+ALTER TABLE gtest27 ALTER COLUMN b TYPE numeric;
+\d gtest27
+ Table "public.gtest27"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+--------------------------------------
+ a | integer | | |
+ b | numeric | | | generated always as ((a * 2)) stored
+
+SELECT * FROM gtest27;
+ a | b
+---+---
+ 3 | 6
+ 4 | 8
+(2 rows)
+
+ALTER TABLE gtest27 ALTER COLUMN b TYPE boolean USING b <> 0; -- error
+ERROR: generation expression for column "b" cannot be cast automatically to type boolean
+ALTER TABLE gtest27 ALTER COLUMN b DROP DEFAULT; -- error
+ERROR: column "b" of relation "gtest27" is a generated column
+\d gtest27
+ Table "public.gtest27"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+--------------------------------------
+ a | integer | | |
+ b | numeric | | | generated always as ((a * 2)) stored
+
+-- triggers
+CREATE TABLE gtest26 (
+ a int PRIMARY KEY,
+ b int GENERATED ALWAYS AS (a * 2) STORED
+);
+CREATE FUNCTION gtest_trigger_func() RETURNS trigger
+ LANGUAGE plpgsql
+AS $$
+BEGIN
+ IF tg_op IN ('DELETE', 'UPDATE') THEN
+ RAISE INFO '%: %: old = %', TG_NAME, TG_WHEN, OLD;
+ END IF;
+ IF tg_op IN ('INSERT', 'UPDATE') THEN
+ RAISE INFO '%: %: new = %', TG_NAME, TG_WHEN, NEW;
+ END IF;
+ IF tg_op = 'DELETE' THEN
+ RETURN OLD;
+ ELSE
+ RETURN NEW;
+ END IF;
+END
+$$;
+CREATE TRIGGER gtest1 BEFORE DELETE OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (OLD.b < 0) -- ok
+ EXECUTE PROCEDURE gtest_trigger_func();
+CREATE TRIGGER gtest2a BEFORE INSERT OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.b < 0) -- error
+ EXECUTE PROCEDURE gtest_trigger_func();
+ERROR: BEFORE trigger's WHEN condition cannot reference NEW generated columns
+LINE 3: WHEN (NEW.b < 0)
+ ^
+DETAIL: Column "b" is a generated column.
+CREATE TRIGGER gtest2b BEFORE INSERT OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.* IS NOT NULL) -- error
+ EXECUTE PROCEDURE gtest_trigger_func();
+ERROR: BEFORE trigger's WHEN condition cannot reference NEW generated columns
+LINE 3: WHEN (NEW.* IS NOT NULL)
+ ^
+DETAIL: A whole-row reference is used and the table contains generated columns.
+CREATE TRIGGER gtest2 BEFORE INSERT ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.a < 0)
+ EXECUTE PROCEDURE gtest_trigger_func();
+CREATE TRIGGER gtest3 AFTER DELETE OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (OLD.b < 0) -- ok
+ EXECUTE PROCEDURE gtest_trigger_func();
+CREATE TRIGGER gtest4 AFTER INSERT OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.b < 0) -- ok
+ EXECUTE PROCEDURE gtest_trigger_func();
+INSERT INTO gtest26 (a) VALUES (-2), (0), (3);
+INFO: gtest2: BEFORE: new = (-2,)
+INFO: gtest4: AFTER: new = (-2,-4)
+SELECT * FROM gtest26 ORDER BY a;
+ a | b
+----+----
+ -2 | -4
+ 0 | 0
+ 3 | 6
+(3 rows)
+
+UPDATE gtest26 SET a = a * -2;
+INFO: gtest1: BEFORE: old = (-2,-4)
+INFO: gtest1: BEFORE: new = (4,)
+INFO: gtest3: AFTER: old = (-2,-4)
+INFO: gtest3: AFTER: new = (4,8)
+INFO: gtest4: AFTER: old = (3,6)
+INFO: gtest4: AFTER: new = (-6,-12)
+SELECT * FROM gtest26 ORDER BY a;
+ a | b
+----+-----
+ -6 | -12
+ 0 | 0
+ 4 | 8
+(3 rows)
+
+DELETE FROM gtest26 WHERE a = -6;
+INFO: gtest1: BEFORE: old = (-6,-12)
+INFO: gtest3: AFTER: old = (-6,-12)
+SELECT * FROM gtest26 ORDER BY a;
+ a | b
+---+---
+ 0 | 0
+ 4 | 8
+(2 rows)
+
+DROP TRIGGER gtest1 ON gtest26;
+DROP TRIGGER gtest2 ON gtest26;
+DROP TRIGGER gtest3 ON gtest26;
+-- Check that an UPDATE of "a" fires the trigger for UPDATE OF b, per
+-- SQL standard.
+CREATE FUNCTION gtest_trigger_func3() RETURNS trigger
+ LANGUAGE plpgsql
+AS $$
+BEGIN
+ RAISE NOTICE 'OK';
+ RETURN NEW;
+END
+$$;
+CREATE TRIGGER gtest11 BEFORE UPDATE OF b ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func3();
+UPDATE gtest26 SET a = 1 WHERE a = 0;
+NOTICE: OK
+DROP TRIGGER gtest11 ON gtest26;
+TRUNCATE gtest26;
+-- check that modifications of stored generated columns in triggers do
+-- not get propagated
+CREATE FUNCTION gtest_trigger_func4() RETURNS trigger
+ LANGUAGE plpgsql
+AS $$
+BEGIN
+ NEW.a = 10;
+ NEW.b = 300;
+ RETURN NEW;
+END;
+$$;
+CREATE TRIGGER gtest12_01 BEFORE UPDATE ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func();
+CREATE TRIGGER gtest12_02 BEFORE UPDATE ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func4();
+CREATE TRIGGER gtest12_03 BEFORE UPDATE ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func();
+INSERT INTO gtest26 (a) VALUES (1);
+UPDATE gtest26 SET a = 11 WHERE a = 1;
+INFO: gtest12_01: BEFORE: old = (1,2)
+INFO: gtest12_01: BEFORE: new = (11,)
+INFO: gtest12_03: BEFORE: old = (1,2)
+INFO: gtest12_03: BEFORE: new = (10,)
+SELECT * FROM gtest26 ORDER BY a;
+ a | b
+----+----
+ 10 | 20
+(1 row)
+
+-- LIKE INCLUDING GENERATED and dropped column handling
+CREATE TABLE gtest28a (
+ a int,
+ b int,
+ c int,
+ x int GENERATED ALWAYS AS (b * 2) STORED
+);
+ALTER TABLE gtest28a DROP COLUMN a;
+CREATE TABLE gtest28b (LIKE gtest28a INCLUDING GENERATED);
+\d gtest28*
+ Table "public.gtest28a"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ b | integer | | |
+ c | integer | | |
+ x | integer | | | generated always as (b * 2) stored
+
+ Table "public.gtest28b"
+ Column | Type | Collation | Nullable | Default
+--------+---------+-----------+----------+------------------------------------
+ b | integer | | |
+ c | integer | | |
+ x | integer | | | generated always as (b * 2) stored
+
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 908fbf650a..f320fb6ef3 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -84,7 +84,7 @@ test: select_into select_distinct select_distinct_on select_implicit select_havi
# ----------
# Another group of parallel tests
# ----------
-test: brin gin gist spgist privileges init_privs security_label collate matview lock replica_identity rowsecurity object_address tablesample groupingsets drop_operator password identity
+test: brin gin gist spgist privileges init_privs security_label collate matview lock replica_identity rowsecurity object_address tablesample groupingsets drop_operator password identity generated
# ----------
# Another group of parallel tests
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index fa754d1c6b..36644aa963 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -122,6 +122,7 @@ test: groupingsets
test: drop_operator
test: password
test: identity
+test: generated
test: create_table_like
test: alter_generic
test: alter_operator
diff --git a/src/test/regress/sql/create_table_like.sql b/src/test/regress/sql/create_table_like.sql
index 65c3880792..9b19c680b5 100644
--- a/src/test/regress/sql/create_table_like.sql
+++ b/src/test/regress/sql/create_table_like.sql
@@ -51,6 +51,20 @@ CREATE TABLE test_like_id_3 (LIKE test_like_id_1 INCLUDING IDENTITY);
SELECT * FROM test_like_id_3; -- identity was copied and applied
DROP TABLE test_like_id_1, test_like_id_2, test_like_id_3;
+CREATE TABLE test_like_gen_1 (a int, b int GENERATED ALWAYS AS (a * 2) STORED);
+\d test_like_gen_1
+INSERT INTO test_like_gen_1 (a) VALUES (1);
+SELECT * FROM test_like_gen_1;
+CREATE TABLE test_like_gen_2 (LIKE test_like_gen_1);
+\d test_like_gen_2
+INSERT INTO test_like_gen_2 (a) VALUES (1);
+SELECT * FROM test_like_gen_2;
+CREATE TABLE test_like_gen_3 (LIKE test_like_gen_1 INCLUDING GENERATED);
+\d test_like_gen_3
+INSERT INTO test_like_gen_3 (a) VALUES (1);
+SELECT * FROM test_like_gen_3;
+DROP TABLE test_like_gen_1, test_like_gen_2, test_like_gen_3;
+
CREATE TABLE inhg (x text, LIKE inhx INCLUDING INDEXES, y text); /* copies indexes */
INSERT INTO inhg VALUES (5, 10);
INSERT INTO inhg VALUES (20, 10); -- should fail
diff --git a/src/test/regress/sql/generated.sql b/src/test/regress/sql/generated.sql
new file mode 100644
index 0000000000..6d71594dc4
--- /dev/null
+++ b/src/test/regress/sql/generated.sql
@@ -0,0 +1,451 @@
+-- sanity check of system catalog
+SELECT attrelid, attname, attgenerated FROM pg_attribute WHERE attgenerated NOT IN ('', 's');
+
+
+CREATE TABLE gtest0 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (55) STORED);
+CREATE TABLE gtest1 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+
+SELECT table_name, column_name, column_default, is_nullable, is_generated, generation_expression FROM information_schema.columns WHERE table_name LIKE 'gtest_' ORDER BY 1, 2;
+
+SELECT table_name, column_name, dependent_column FROM information_schema.column_column_usage ORDER BY 1, 2, 3;
+
+\d gtest1
+
+-- duplicate generated
+CREATE TABLE gtest_err_1 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED GENERATED ALWAYS AS (a * 3) STORED);
+
+-- references to other generated columns, including self-references
+CREATE TABLE gtest_err_2a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (b * 2) STORED);
+CREATE TABLE gtest_err_2b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED, c int GENERATED ALWAYS AS (b * 3) STORED);
+
+-- invalid reference
+CREATE TABLE gtest_err_3 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (c * 2) STORED);
+
+-- generation expression must be immutable
+CREATE TABLE gtest_err_4 (a int PRIMARY KEY, b double precision GENERATED ALWAYS AS (random()) STORED);
+
+-- cannot have default/identity and generated
+CREATE TABLE gtest_err_5a (a int PRIMARY KEY, b int DEFAULT 5 GENERATED ALWAYS AS (a * 2) STORED);
+CREATE TABLE gtest_err_5b (a int PRIMARY KEY, b int GENERATED ALWAYS AS identity GENERATED ALWAYS AS (a * 2) STORED);
+
+-- reference to system column not allowed in generated column
+CREATE TABLE gtest_err_6a (a int PRIMARY KEY, b bool GENERATED ALWAYS AS (xmin <> 37) STORED);
+
+-- various prohibited constructs
+CREATE TABLE gtest_err_7a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (avg(a)) STORED);
+CREATE TABLE gtest_err_7b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (row_number() OVER (ORDER BY a)) STORED);
+CREATE TABLE gtest_err_7c (a int PRIMARY KEY, b int GENERATED ALWAYS AS ((SELECT a)) STORED);
+CREATE TABLE gtest_err_7d (a int PRIMARY KEY, b int GENERATED ALWAYS AS (generate_series(1, a)) STORED);
+
+INSERT INTO gtest1 VALUES (1);
+INSERT INTO gtest1 VALUES (2, DEFAULT);
+INSERT INTO gtest1 VALUES (3, 33); -- error
+
+SELECT * FROM gtest1 ORDER BY a;
+
+UPDATE gtest1 SET b = DEFAULT WHERE a = 1;
+UPDATE gtest1 SET b = 11 WHERE a = 1; -- error
+
+SELECT * FROM gtest1 ORDER BY a;
+
+SELECT a, b, b * 2 AS b2 FROM gtest1 ORDER BY a;
+SELECT a, b FROM gtest1 WHERE b = 4 ORDER BY a;
+
+-- test that overflow error happens on write
+INSERT INTO gtest1 VALUES (2000000000);
+SELECT * FROM gtest1;
+DELETE FROM gtest1 WHERE a = 2000000000;
+
+-- test with joins
+CREATE TABLE gtestx (x int, y int);
+INSERT INTO gtestx VALUES (11, 1), (22, 2), (33, 3);
+SELECT * FROM gtestx, gtest1 WHERE gtestx.y = gtest1.a;
+DROP TABLE gtestx;
+
+-- test UPDATE/DELETE quals
+SELECT * FROM gtest1 ORDER BY a;
+UPDATE gtest1 SET a = 3 WHERE b = 4;
+SELECT * FROM gtest1 ORDER BY a;
+DELETE FROM gtest1 WHERE b = 2;
+SELECT * FROM gtest1 ORDER BY a;
+
+-- views
+CREATE VIEW gtest1v AS SELECT * FROM gtest1;
+SELECT * FROM gtest1v;
+INSERT INTO gtest1v VALUES (4, 8); -- fails
+DROP VIEW gtest1v;
+
+-- CTEs
+WITH foo AS (SELECT * FROM gtest1) SELECT * FROM foo;
+
+-- inheritance
+CREATE TABLE gtest1_1 () INHERITS (gtest1);
+SELECT * FROM gtest1_1;
+\d gtest1_1
+INSERT INTO gtest1_1 VALUES (4);
+SELECT * FROM gtest1_1;
+SELECT * FROM gtest1;
+
+-- test inheritance mismatch
+CREATE TABLE gtesty (x int, b int);
+CREATE TABLE gtest1_2 () INHERITS (gtest1, gtesty); -- error
+DROP TABLE gtesty;
+
+-- test stored update
+CREATE TABLE gtest3 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 3) STORED);
+INSERT INTO gtest3 (a) VALUES (1), (2), (3);
+SELECT * FROM gtest3 ORDER BY a;
+UPDATE gtest3 SET a = 22 WHERE a = 2;
+SELECT * FROM gtest3 ORDER BY a;
+
+-- COPY
+TRUNCATE gtest1;
+INSERT INTO gtest1 (a) VALUES (1), (2);
+
+COPY gtest1 TO stdout;
+
+COPY gtest1 (a, b) TO stdout;
+
+COPY gtest1 FROM stdin;
+3
+4
+\.
+
+COPY gtest1 (a, b) FROM stdin;
+
+SELECT * FROM gtest1 ORDER BY a;
+
+TRUNCATE gtest3;
+INSERT INTO gtest3 (a) VALUES (1), (2);
+
+COPY gtest3 TO stdout;
+
+COPY gtest3 (a, b) TO stdout;
+
+COPY gtest3 FROM stdin;
+3
+4
+\.
+
+COPY gtest3 (a, b) FROM stdin;
+
+SELECT * FROM gtest3 ORDER BY a;
+
+-- null values
+CREATE TABLE gtest2 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (NULL) STORED);
+INSERT INTO gtest2 VALUES (1);
+SELECT * FROM gtest2;
+
+-- composite types
+CREATE TYPE double_int as (a int, b int);
+CREATE TABLE gtest4 (
+ a int,
+ b double_int GENERATED ALWAYS AS ((a * 2, a * 3)) STORED
+);
+INSERT INTO gtest4 VALUES (1), (6);
+SELECT * FROM gtest4;
+
+DROP TABLE gtest4;
+DROP TYPE double_int;
+
+-- using tableoid is allowed
+CREATE TABLE gtest_tableoid (
+ a int PRIMARY KEY,
+ b bool GENERATED ALWAYS AS (tableoid <> 0) STORED
+);
+INSERT INTO gtest_tableoid VALUES (1), (2);
+SELECT * FROM gtest_tableoid;
+
+-- drop column behavior
+CREATE TABLE gtest10 (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (b * 2) STORED);
+ALTER TABLE gtest10 DROP COLUMN b;
+
+\d gtest10
+
+CREATE TABLE gtest10a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+ALTER TABLE gtest10a DROP COLUMN b;
+INSERT INTO gtest10a (a) VALUES (1);
+
+-- privileges
+CREATE USER regress_user11;
+
+CREATE TABLE gtest11s (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (b * 2) STORED);
+INSERT INTO gtest11s VALUES (1, 10), (2, 20);
+GRANT SELECT (a, c) ON gtest11s TO regress_user11;
+
+CREATE FUNCTION gf1(a int) RETURNS int AS $$ SELECT a * 3 $$ IMMUTABLE LANGUAGE SQL;
+REVOKE ALL ON FUNCTION gf1(int) FROM PUBLIC;
+
+CREATE TABLE gtest12s (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (gf1(b)) STORED);
+INSERT INTO gtest12s VALUES (1, 10), (2, 20);
+GRANT SELECT (a, c) ON gtest12s TO regress_user11;
+
+SET ROLE regress_user11;
+SELECT a, b FROM gtest11s; -- not allowed
+SELECT a, c FROM gtest11s; -- allowed
+SELECT gf1(10); -- not allowed
+SELECT a, c FROM gtest12s; -- allowed
+RESET ROLE;
+
+DROP TABLE gtest11s, gtest12s;
+DROP FUNCTION gf1(int);
+DROP USER regress_user11;
+
+-- check constraints
+CREATE TABLE gtest20 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED CHECK (b < 50));
+INSERT INTO gtest20 (a) VALUES (10); -- ok
+INSERT INTO gtest20 (a) VALUES (30); -- violates constraint
+
+CREATE TABLE gtest20a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+INSERT INTO gtest20a (a) VALUES (10);
+INSERT INTO gtest20a (a) VALUES (30);
+ALTER TABLE gtest20a ADD CHECK (b < 50); -- fails on existing row
+
+CREATE TABLE gtest20b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED);
+INSERT INTO gtest20b (a) VALUES (10);
+INSERT INTO gtest20b (a) VALUES (30);
+ALTER TABLE gtest20b ADD CONSTRAINT chk CHECK (b < 50) NOT VALID;
+ALTER TABLE gtest20b VALIDATE CONSTRAINT chk; -- fails on existing row
+
+-- not-null constraints
+CREATE TABLE gtest21a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (nullif(a, 0)) STORED NOT NULL);
+INSERT INTO gtest21a (a) VALUES (1); -- ok
+INSERT INTO gtest21a (a) VALUES (0); -- violates constraint
+
+CREATE TABLE gtest21b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (nullif(a, 0)) STORED);
+ALTER TABLE gtest21b ALTER COLUMN b SET NOT NULL;
+INSERT INTO gtest21b (a) VALUES (1); -- ok
+INSERT INTO gtest21b (a) VALUES (0); -- violates constraint
+ALTER TABLE gtest21b ALTER COLUMN b DROP NOT NULL;
+INSERT INTO gtest21b (a) VALUES (0); -- ok now
+
+-- index constraints
+CREATE TABLE gtest22a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a / 2) STORED UNIQUE);
+INSERT INTO gtest22a VALUES (2);
+INSERT INTO gtest22a VALUES (3);
+INSERT INTO gtest22a VALUES (4);
+CREATE TABLE gtest22b (a int, b int GENERATED ALWAYS AS (a / 2) STORED, PRIMARY KEY (a, b));
+INSERT INTO gtest22b VALUES (2);
+INSERT INTO gtest22b VALUES (2);
+
+-- indexes
+CREATE TABLE gtest22c (a int, b int GENERATED ALWAYS AS (a * 2) STORED);
+CREATE INDEX gtest22c_b_idx ON gtest22c (b);
+CREATE INDEX gtest22c_expr_idx ON gtest22c ((b * 3));
+CREATE INDEX gtest22c_pred_idx ON gtest22c (a) WHERE b > 0;
+\d gtest22c
+
+INSERT INTO gtest22c VALUES (1), (2), (3);
+SET enable_seqscan TO off;
+SET enable_bitmapscan TO off;
+EXPLAIN (COSTS OFF) SELECT * FROM gtest22c WHERE b = 4;
+SELECT * FROM gtest22c WHERE b = 4;
+EXPLAIN (COSTS OFF) SELECT * FROM gtest22c WHERE b * 3 = 6;
+SELECT * FROM gtest22c WHERE b * 3 = 6;
+EXPLAIN (COSTS OFF) SELECT * FROM gtest22c WHERE a = 1 AND b > 0;
+SELECT * FROM gtest22c WHERE a = 1 AND b > 0;
+RESET enable_seqscan;
+RESET enable_bitmapscan;
+
+-- foreign keys
+CREATE TABLE gtest23a (x int PRIMARY KEY, y int);
+INSERT INTO gtest23a VALUES (1, 11), (2, 22), (3, 33);
+
+CREATE TABLE gtest23x (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED REFERENCES gtest23a (x) ON UPDATE CASCADE); -- error
+CREATE TABLE gtest23x (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED REFERENCES gtest23a (x) ON DELETE SET NULL); -- error
+
+CREATE TABLE gtest23b (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED REFERENCES gtest23a (x));
+\d gtest23b
+
+INSERT INTO gtest23b VALUES (1); -- ok
+INSERT INTO gtest23b VALUES (5); -- error
+
+DROP TABLE gtest23b;
+DROP TABLE gtest23a;
+
+CREATE TABLE gtest23p (x int, y int GENERATED ALWAYS AS (x * 2) STORED, PRIMARY KEY (y));
+INSERT INTO gtest23p VALUES (1), (2), (3);
+
+CREATE TABLE gtest23q (a int PRIMARY KEY, b int REFERENCES gtest23p (y));
+INSERT INTO gtest23q VALUES (1, 2); -- ok
+INSERT INTO gtest23q VALUES (2, 5); -- error
+
+-- domains
+CREATE DOMAIN gtestdomain1 AS int CHECK (VALUE < 10);
+CREATE TABLE gtest24 (a int PRIMARY KEY, b gtestdomain1 GENERATED ALWAYS AS (a * 2) STORED);
+INSERT INTO gtest24 (a) VALUES (4); -- ok
+INSERT INTO gtest24 (a) VALUES (6); -- error
+
+-- typed tables (currently not supported)
+CREATE TYPE gtest_type AS (f1 integer, f2 text, f3 bigint);
+CREATE TABLE gtest28 OF gtest_type (f1 WITH OPTIONS GENERATED ALWAYS AS (f2 *2) STORED);
+DROP TYPE gtest_type CASCADE;
+
+-- table partitions (currently not supported)
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 text, f3 bigint) PARTITION BY RANGE (f1);
+CREATE TABLE gtest_child PARTITION OF gtest_parent (
+ f3 WITH OPTIONS GENERATED ALWAYS AS (f2 * 2) STORED
+) FOR VALUES FROM ('2016-07-01') TO ('2016-08-01'); -- error
+DROP TABLE gtest_parent;
+
+-- partitioned table
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 bigint, f3 bigint GENERATED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE (f1);
+CREATE TABLE gtest_child PARTITION OF gtest_parent FOR VALUES FROM ('2016-07-01') TO ('2016-08-01');
+INSERT INTO gtest_parent (f1, f2) VALUES ('2016-07-15', 1);
+SELECT * FROM gtest_parent;
+SELECT * FROM gtest_child;
+DROP TABLE gtest_parent;
+
+-- generated columns in partition key (not allowed)
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 bigint, f3 bigint GENERATED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE (f3);
+CREATE TABLE gtest_parent (f1 date NOT NULL, f2 bigint, f3 bigint GENERATED ALWAYS AS (f2 * 2) STORED) PARTITION BY RANGE ((f3 * 3));
+
+-- ALTER TABLE ... ADD COLUMN
+CREATE TABLE gtest25 (a int PRIMARY KEY);
+INSERT INTO gtest25 VALUES (3), (4);
+ALTER TABLE gtest25 ADD COLUMN b int GENERATED ALWAYS AS (a * 3) STORED;
+SELECT * FROM gtest25 ORDER BY a;
+ALTER TABLE gtest25 ADD COLUMN x int GENERATED ALWAYS AS (b * 4) STORED; -- error
+ALTER TABLE gtest25 ADD COLUMN x int GENERATED ALWAYS AS (z * 4) STORED; -- error
+
+-- ALTER TABLE ... ALTER COLUMN
+CREATE TABLE gtest27 (
+ a int,
+ b int GENERATED ALWAYS AS (a * 2) STORED
+);
+INSERT INTO gtest27 (a) VALUES (3), (4);
+ALTER TABLE gtest27 ALTER COLUMN a TYPE text; -- error
+ALTER TABLE gtest27 ALTER COLUMN b TYPE numeric;
+\d gtest27
+SELECT * FROM gtest27;
+ALTER TABLE gtest27 ALTER COLUMN b TYPE boolean USING b <> 0; -- error
+
+ALTER TABLE gtest27 ALTER COLUMN b DROP DEFAULT; -- error
+\d gtest27
+
+-- triggers
+CREATE TABLE gtest26 (
+ a int PRIMARY KEY,
+ b int GENERATED ALWAYS AS (a * 2) STORED
+);
+
+CREATE FUNCTION gtest_trigger_func() RETURNS trigger
+ LANGUAGE plpgsql
+AS $$
+BEGIN
+ IF tg_op IN ('DELETE', 'UPDATE') THEN
+ RAISE INFO '%: %: old = %', TG_NAME, TG_WHEN, OLD;
+ END IF;
+ IF tg_op IN ('INSERT', 'UPDATE') THEN
+ RAISE INFO '%: %: new = %', TG_NAME, TG_WHEN, NEW;
+ END IF;
+ IF tg_op = 'DELETE' THEN
+ RETURN OLD;
+ ELSE
+ RETURN NEW;
+ END IF;
+END
+$$;
+
+CREATE TRIGGER gtest1 BEFORE DELETE OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (OLD.b < 0) -- ok
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+CREATE TRIGGER gtest2a BEFORE INSERT OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.b < 0) -- error
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+CREATE TRIGGER gtest2b BEFORE INSERT OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.* IS NOT NULL) -- error
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+CREATE TRIGGER gtest2 BEFORE INSERT ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.a < 0)
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+CREATE TRIGGER gtest3 AFTER DELETE OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (OLD.b < 0) -- ok
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+CREATE TRIGGER gtest4 AFTER INSERT OR UPDATE ON gtest26
+ FOR EACH ROW
+ WHEN (NEW.b < 0) -- ok
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+INSERT INTO gtest26 (a) VALUES (-2), (0), (3);
+SELECT * FROM gtest26 ORDER BY a;
+UPDATE gtest26 SET a = a * -2;
+SELECT * FROM gtest26 ORDER BY a;
+DELETE FROM gtest26 WHERE a = -6;
+SELECT * FROM gtest26 ORDER BY a;
+
+DROP TRIGGER gtest1 ON gtest26;
+DROP TRIGGER gtest2 ON gtest26;
+DROP TRIGGER gtest3 ON gtest26;
+
+-- Check that an UPDATE of "a" fires the trigger for UPDATE OF b, per
+-- SQL standard.
+CREATE FUNCTION gtest_trigger_func3() RETURNS trigger
+ LANGUAGE plpgsql
+AS $$
+BEGIN
+ RAISE NOTICE 'OK';
+ RETURN NEW;
+END
+$$;
+
+CREATE TRIGGER gtest11 BEFORE UPDATE OF b ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func3();
+
+UPDATE gtest26 SET a = 1 WHERE a = 0;
+
+DROP TRIGGER gtest11 ON gtest26;
+TRUNCATE gtest26;
+
+-- check that modifications of stored generated columns in triggers do
+-- not get propagated
+CREATE FUNCTION gtest_trigger_func4() RETURNS trigger
+ LANGUAGE plpgsql
+AS $$
+BEGIN
+ NEW.a = 10;
+ NEW.b = 300;
+ RETURN NEW;
+END;
+$$;
+
+CREATE TRIGGER gtest12_01 BEFORE UPDATE ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+CREATE TRIGGER gtest12_02 BEFORE UPDATE ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func4();
+
+CREATE TRIGGER gtest12_03 BEFORE UPDATE ON gtest26
+ FOR EACH ROW
+ EXECUTE PROCEDURE gtest_trigger_func();
+
+INSERT INTO gtest26 (a) VALUES (1);
+UPDATE gtest26 SET a = 11 WHERE a = 1;
+SELECT * FROM gtest26 ORDER BY a;
+
+-- LIKE INCLUDING GENERATED and dropped column handling
+CREATE TABLE gtest28a (
+ a int,
+ b int,
+ c int,
+ x int GENERATED ALWAYS AS (b * 2) STORED
+);
+
+ALTER TABLE gtest28a DROP COLUMN a;
+
+CREATE TABLE gtest28b (LIKE gtest28a INCLUDING GENERATED);
+
+\d gtest28*
diff --git a/src/test/subscription/t/011_generated.pl b/src/test/subscription/t/011_generated.pl
new file mode 100644
index 0000000000..f7456e9216
--- /dev/null
+++ b/src/test/subscription/t/011_generated.pl
@@ -0,0 +1,65 @@
+# Test generated columns
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 2;
+
+# setup
+
+my $node_publisher = get_new_node('publisher');
+$node_publisher->init(allows_streaming => 'logical');
+$node_publisher->start;
+
+my $node_subscriber = get_new_node('subscriber');
+$node_subscriber->init(allows_streaming => 'logical');
+$node_subscriber->start;
+
+my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres';
+
+$node_publisher->safe_psql('postgres',
+ "CREATE TABLE tab1 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STORED)");
+
+$node_subscriber->safe_psql('postgres',
+ "CREATE TABLE tab1 (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 22) STORED)");
+
+# data for initial sync
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab1 (a) VALUES (1), (2), (3)");
+
+$node_publisher->safe_psql('postgres',
+ "CREATE PUBLICATION pub1 FOR ALL TABLES");
+$node_subscriber->safe_psql('postgres',
+ "CREATE SUBSCRIPTION sub1 CONNECTION '$publisher_connstr' PUBLICATION pub1"
+);
+
+# Wait for initial sync of all subscriptions
+my $synced_query =
+ "SELECT count(1) = 0 FROM pg_subscription_rel WHERE srsubstate NOT IN ('r', 's');";
+$node_subscriber->poll_query_until('postgres', $synced_query)
+ or die "Timed out while waiting for subscriber to synchronize data";
+
+my $result = $node_subscriber->safe_psql('postgres',
+ "SELECT a, b FROM tab1");
+is($result, qq(1|22
+2|44
+3|66), 'generated columns initial sync');
+
+# data to replicate
+
+$node_publisher->safe_psql('postgres',
+ "INSERT INTO tab1 VALUES (4), (5)");
+
+$node_publisher->safe_psql('postgres',
+ "UPDATE tab1 SET a = 6 WHERE a = 5");
+
+$node_publisher->wait_for_catchup('sub1');
+
+$result = $node_subscriber->safe_psql('postgres',
+ "SELECT a, b FROM tab1");
+is($result, qq(1|22
+2|44
+3|66
+4|88
+6|132), 'generated columns replicated');
base-commit: c8c885b7a5c8c1175288de1d8aaec3b4ae9050e1
--
2.21.0
--------------A25BEA11677B46AF0D81EBB9--
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v7 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 272 ++++++++++++++----------
src/backend/access/transam/xlogutils.c | 12 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
8 files changed, 211 insertions(+), 148 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7621fc05e2..51c409d00e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -900,7 +900,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
XLogSource source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, XLogSource source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4285,7 +4285,6 @@ ReadRecord(XLogReaderState *xlogreader, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (xlogreader->ReadRecPtr == InvalidXLogRecPtr);
@@ -11660,7 +11659,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11719,7 +11718,8 @@ retry:
readLen = 0;
readSource = XLOG_FROM_ANY;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11814,7 +11814,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11828,8 +11829,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 32f02256ed..2c1500443e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -269,7 +269,6 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -319,14 +318,20 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -352,8 +357,8 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -426,18 +431,25 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -466,21 +478,14 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -510,9 +515,15 @@ XLogReadRecord(XLogReaderState *state, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -555,109 +566,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -666,9 +707,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -949,7 +988,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -957,7 +995,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -965,19 +1003,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index b217ffa52f..47676bf800 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -685,8 +685,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -818,7 +818,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext * segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -920,7 +920,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -938,7 +939,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 76ec3c7dd0..c66ea308d8 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index eb61cb8803..78ee9f3faa 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -227,7 +227,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -282,7 +282,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -295,7 +296,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -308,13 +310,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 279acfa044..443fe33599 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -322,7 +322,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -339,7 +339,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -364,7 +365,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 4582196e18..6ad953eea3 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -51,7 +51,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -134,6 +134,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -160,13 +174,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 5181a077d9..dc7d894e5d 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
--
2.18.2
----Next_Part(Tue_Mar_24_18_24_13_2020_275)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v7-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v12 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 272 ++++++++++--------
src/backend/access/transam/xlogutils.c | 12 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 213 insertions(+), 150 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5f0ee50092..4a6bd6e002 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11537,7 +11536,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11596,7 +11595,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11691,7 +11691,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11705,8 +11706,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 67418b05f1..063223701e 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,14 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -330,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -404,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -444,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -488,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -533,109 +544,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -644,9 +685,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -924,7 +963,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -932,7 +970,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * XLogNeedData() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -940,19 +978,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 446760ed6e..060fbc0c4c 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -680,8 +680,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -813,7 +813,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -915,7 +915,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -933,7 +934,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index aa2106b152..4b0e8ea773 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -106,7 +106,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 8bafa65e50..554c186ae1 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 1d03375a3e..795e84ff9f 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -228,7 +228,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -283,7 +283,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -296,7 +297,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -309,13 +311,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 30a5851d87..df13cf3173 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,7 +324,7 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
+static bool
WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -341,7 +341,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -366,7 +367,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 0193611b7f..d990432ffe 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -49,7 +49,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -131,6 +131,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -157,13 +171,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0572b24192..0867ef0599 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.23.0
----Next_Part(Fri_Nov_29_17_14_21_2019_330)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v12-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v7 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 332 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 259 insertions(+), 162 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6c69eb6dd7..5dcb2e500c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11522,7 +11521,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11581,7 +11580,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11676,7 +11676,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11690,8 +11691,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 27c27303d6..c2bb664f07 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,9 +34,9 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
-static void XLogReaderInvalReadState(XLogReaderState *state);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
+static void XLogReaderDiscardReadingPage(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
@@ -104,7 +104,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,15 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -331,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -405,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -445,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -489,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -525,7 +535,7 @@ err:
* Invalidate the read state. We might read from a different source after
* failure.
*/
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
@@ -534,120 +544,183 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->seg.ws_off && reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (!header_inclusive)
+ {
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Haven't loaded any data yet? Then request it. */
+ if (XLogRecPtrIsInvalid(state->readPagePtr) ||
+ state->readPagePtr != pageptr || state->readLen < 0)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen, SizeOfXLogShortPHD);
+ state->page_verified = false;
+ return true;
+ }
+
+ if (!state->page_verified)
+ {
+ uint32 pageHeaderSize;
+ uint32 addLen = 0;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /*
+ * If we have not loaded a page so far, readLen is zero, which is
+ * shorter than pageHeaderSize here.
+ */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /*
+ * Now that we know we have the full header, validate it.
+ */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* force reading the page again. */
+ XLogReaderDiscardReadingPage(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ if (!header_inclusive)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Usually we have requested data loaded in buffer here. */
+ if (pageptr == state->readPagePtr && reqLen + addLen <= state->readLen)
+ return false;
+
+ /*
+ * In the case the page is requested for the first record in the page,
+ * the previous request have been made not counting page header
+ * length. Request again for the same page with the length known to be
+ * needed. Otherwise we don't know the header length of the new page.
+ */
+ if (pageptr != state->readPagePtr)
+ addLen = 0;
+ }
+
+ /* Data is not in our buffer, make a new load request to the caller. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the requested page. We need at least a short
+ * page header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->seg.ws_off = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
- * Invalidate the xlogreader's read state to force a re-read.
+ * Invalidate current reading page buffer
*/
static void
-XLogReaderInvalReadState(XLogReaderState *state)
+XLogReaderDiscardReadingPage(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->seg.ws_off = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -925,7 +998,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -933,7 +1005,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -941,19 +1013,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
@@ -1010,7 +1086,7 @@ out:
/* Reset state to what we had before finding the record */
state->ReadRecPtr = saved_state.ReadRecPtr;
state->EndRecPtr = saved_state.EndRecPtr;
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
return found;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5f1e5ba75d..a19726a96e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->seg.ws_off;
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -1007,7 +1007,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1024,5 +1025,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d1cf80d441..310cd9d8cf 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,7 +114,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index eb4a98cc91..0809ceaeb8 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -760,7 +760,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -778,7 +778,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -788,7 +791,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(sendCxt, cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 264a8f4db5..8aecd1adc7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -230,7 +230,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -285,7 +285,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -298,7 +299,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -311,13 +313,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b79208cd73..6e424bd8e1 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -406,7 +406,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -422,14 +422,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1bbee386e8..8b747d465f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,7 +50,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -132,6 +132,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -158,13 +172,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 2df98e45b2..47b65463f9 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.16.3
----Next_Part(Wed_Sep_25_15_50_32_2019_576)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v7-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v8 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 306 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
src/test/recovery/t/011_crash_recovery.pl | 1 +
11 files changed, 239 insertions(+), 157 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6c69eb6dd7..5dcb2e500c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11522,7 +11521,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11581,7 +11580,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11676,7 +11676,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11690,8 +11691,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 27c27303d6..900a628752 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,8 +34,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -104,7 +104,7 @@ XLogReaderAllocate(int wal_segment_size, const char *waldir,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,15 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -331,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -405,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -445,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -489,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -534,109 +544,158 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->seg.ws_off && reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (!header_inclusive)
+ {
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
+
+ /*
+ * The loaded page may not be the one caller is supposing to read when
+ * we are verifying the first page of new segment. In that case, skip
+ * further verification and immediately load the target page.
+ */
+ if (pageptr == state->readPagePtr)
+ {
+
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+ }
+
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->seg.ws_off = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -645,9 +704,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->seg.ws_off = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -925,7 +982,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -933,7 +989,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -941,19 +997,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5f1e5ba75d..a19726a96e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->seg.ws_off;
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -1007,7 +1007,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1024,5 +1025,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d1cf80d441..310cd9d8cf 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,7 +114,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index eb4a98cc91..0809ceaeb8 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -760,7 +760,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -778,7 +778,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -788,7 +791,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(sendCxt, cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 264a8f4db5..8aecd1adc7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -230,7 +230,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -285,7 +285,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -298,7 +299,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -311,13 +313,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b79208cd73..6e424bd8e1 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -406,7 +406,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -422,14 +422,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1bbee386e8..8b747d465f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,7 +50,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -132,6 +132,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -158,13 +172,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 2df98e45b2..47b65463f9 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..c78912571e 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -55,6 +55,7 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
# Crash and restart the postmaster
$node->stop('immediate');
+print "HOGEEEEEEEEEEEE\n";
$node->start;
# Make sure we really got a new xid
--
2.16.3
----Next_Part(Fri_Sep_27_12_07_26_2019_308)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v8-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v9 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 273 ++++++++++--------
src/backend/access/transam/xlogutils.c | 10 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
src/test/recovery/t/011_crash_recovery.pl | 1 +
11 files changed, 213 insertions(+), 150 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index b602e28f61..a7630c643a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11540,7 +11539,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11599,7 +11598,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11694,7 +11694,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11708,8 +11709,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index c8b0d2303d..66f3bc5597 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,8 +34,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -244,7 +244,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -296,15 +295,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -330,8 +334,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -404,18 +408,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -444,21 +455,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -488,9 +492,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -533,109 +543,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->seg.ws_off && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->seg.ws_off = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -644,9 +684,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->seg.ws_off = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -924,7 +962,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -932,7 +969,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -940,19 +977,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 5f1e5ba75d..a19726a96e 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->seg.ws_segno *
- state->segcxt.ws_segsize + state->seg.ws_off;
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -1007,7 +1007,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1024,5 +1025,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d1cf80d441..310cd9d8cf 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,7 +114,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index b0ebe5039c..3ea71a0f84 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -760,7 +760,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -778,7 +778,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -788,7 +791,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(sendCxt, cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 264a8f4db5..8aecd1adc7 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -46,7 +46,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -230,7 +230,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -285,7 +285,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -298,7 +299,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -311,13 +313,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b79208cd73..6e424bd8e1 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -406,7 +406,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff)
{
@@ -422,14 +422,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(state->segcxt.ws_dir, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 1bbee386e8..8b747d465f 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -50,7 +50,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -132,6 +132,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -158,13 +172,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 2df98e45b2..47b65463f9 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
diff --git a/src/test/recovery/t/011_crash_recovery.pl b/src/test/recovery/t/011_crash_recovery.pl
index 526a3481fb..c78912571e 100644
--- a/src/test/recovery/t/011_crash_recovery.pl
+++ b/src/test/recovery/t/011_crash_recovery.pl
@@ -55,6 +55,7 @@ is($node->safe_psql('postgres', qq[SELECT txid_status('$xid');]),
# Crash and restart the postmaster
$node->stop('immediate');
+print "HOGEEEEEEEEEEEE\n";
$node->start;
# Make sure we really got a new xid
--
2.23.0
----Next_Part(Thu_Oct_24_14_51_01_2019_720)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v9-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v6 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 336 +++++++++++++++----------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 4 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 17 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 26 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 265 insertions(+), 166 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6876537b62..d7f899e738 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *readTLI);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11521,7 +11520,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
{
@@ -11580,7 +11579,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11675,7 +11675,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11689,8 +11690,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index a66e3324b1..12a52159a9 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -34,9 +34,9 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
-static void XLogReaderInvalReadState(XLogReaderState *state);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
+static void XLogReaderDiscardReadingPage(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
@@ -101,7 +101,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -225,7 +225,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -277,15 +276,21 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -311,8 +316,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -385,18 +390,26 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -425,21 +438,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -469,9 +475,16 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -505,7 +518,7 @@ err:
* Invalidate the read state. We might read from a different source after
* failure.
*/
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
@@ -514,120 +527,183 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
- targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->readSegNo && targetPageOff == state->readOff &&
- reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (!header_inclusive)
+ {
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Haven't loaded any data yet? Then request it. */
+ if (XLogRecPtrIsInvalid(state->readPagePtr) ||
+ state->readPagePtr != pageptr || state->readLen < 0)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen, SizeOfXLogShortPHD);
+ state->page_verified = false;
+ return true;
+ }
+
+ if (!state->page_verified)
+ {
+ uint32 pageHeaderSize;
+ uint32 addLen = 0;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /*
+ * If we have not loaded a page so far, readLen is zero, which is
+ * shorter than pageHeaderSize here.
+ */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /*
+ * Now that we know we have the full header, validate it.
+ */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* force reading the page again. */
+ XLogReaderDiscardReadingPage(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->readSegNo,
+ state->wal_segment_size);
+
+ /*
+ * calculate additional length for page header so that the total
+ * length doesn't exceed the block size.
+ */
+ if (!header_inclusive)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Usually we have requested data loaded in buffer here. */
+ if (pageptr == state->readPagePtr && reqLen + addLen <= state->readLen)
+ return false;
+
+ /*
+ * In the case the page is requested for the first record in the page,
+ * the previous request have been made not counting page header
+ * length. Request again for the same page with the length known to be
+ * needed. Otherwise we don't know the header length of the new page.
+ */
+ if (pageptr != state->readPagePtr)
+ addLen = 0;
+ }
+
+ /* Data is not in our buffer, make a new load request to the caller. */
+ XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
+ targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->readSegNo && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the readSegNo, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the requested page. We need at least a short
+ * page header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->readSegNo = targetSegNo;
- state->readOff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
- * Invalidate the xlogreader's read state to force a re-read.
+ * Invalidate current reading page buffer
*/
static void
-XLogReaderInvalReadState(XLogReaderState *state)
+XLogReaderDiscardReadingPage(XLogReaderState *state)
{
- state->readSegNo = 0;
- state->readOff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -905,7 +981,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -913,7 +988,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -921,19 +996,24 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
@@ -990,7 +1070,7 @@ out:
/* Reset state to what we had before finding the record */
state->ReadRecPtr = saved_state.ReadRecPtr;
state->EndRecPtr = saved_state.EndRecPtr;
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
return found;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1fc39333f1..dad9074b9f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->readSegNo *
- state->wal_segment_size + state->readOff;
+ state->wal_segment_size + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI)
@@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d974400d6e..7210a940bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,12 +114,12 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ targetRecPtr, cur_page, pageTLI);
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23870a25a5..cc35e2a04d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
@@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 63c3879ead..4df53964e4 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
@@ -235,7 +235,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI)
@@ -290,7 +290,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -303,7 +304,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -316,13 +318,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
*pageTLI = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b95d467805..96d1f36ebc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
{
@@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 735b1bd2fd..6de7c19a2a 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -34,7 +34,7 @@
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -123,6 +123,18 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
/* ----------------------------------------
* Decoded representation of current record
@@ -149,17 +161,9 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
- /* last read segment, segment offset, TLI for data currently in readBuf */
+ /* last read segment and segment offset for data currently in readBuf */
+ bool page_verified;
XLogSegNo readSegNo;
- uint32 readOff;
- TimeLineID readPageTLI;
/*
* beginning of prior page read, and its TLI. Doesn't necessarily
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..0842af9f95 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index a9c178a9e6..8e52b1f4aa 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, TimeLineID *pageTLI);
--
2.16.3
----Next_Part(Tue_Sep_10_17_40_54_2019_177)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v6-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v11 1/4] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
The current WAL record reader reads page data using a call back
function. Although it is not so problematic alone, it would be a
problem if we are going to do add tasks like encryption which is
performed on page data before WAL reader reads them. To avoid that the
record reader facility has to have a new code path corresponds to
every new callback, this patch separates page reader from WAL record
reading facility by modifying the current WAL record reader to a state
machine.
As the first step of that change, this patch moves the page reader
function out of ReadPageInternal, then the remaining tasks of the
function are taken over by the new function XLogNeedData. As the
result XLogPageRead directly calls the page reader callback function
according to the feedback from XLogNeedData.
---
src/backend/access/transam/xlog.c | 16 +-
src/backend/access/transam/xlogreader.c | 272 ++++++++++--------
src/backend/access/transam/xlogutils.c | 12 +-
.../replication/logical/logicalfuncs.c | 2 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 16 +-
src/bin/pg_waldump/pg_waldump.c | 14 +-
src/include/access/xlogreader.h | 23 +-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 216 insertions(+), 153 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 5f0ee50092..4a6bd6e002 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
bool fetching_ckpt, XLogRecPtr tliRecPtr);
@@ -4249,7 +4249,6 @@ ReadRecord(XLogReaderState *xlogreader, XLogRecPtr RecPtr, int emode,
XLogRecord *record;
XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
- /* Pass through parameters to XLogPageRead */
private->fetching_ckpt = fetching_ckpt;
private->emode = emode;
private->randAccess = (RecPtr != InvalidXLogRecPtr);
@@ -11537,7 +11536,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -11596,7 +11595,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11691,7 +11691,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11705,8 +11706,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index 67418b05f1..388662ade4 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@
static void report_invalid_record(XLogReaderState *state, const char *fmt,...)
pg_attribute_printf(2, 3);
static bool allocate_recordbuf(XLogReaderState *state, uint32 reclength);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool header_inclusive);
static void XLogReaderInvalReadState(XLogReaderState *state);
static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
@@ -245,7 +245,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -297,14 +296,20 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ RecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -330,8 +335,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -404,18 +409,25 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
do
{
+ int rest_len = total_len - gotlen;
+
/* Calculate pointer to beginning of next page */
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(rest_len, XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -444,21 +456,14 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
-
- Assert(pageHeaderSize <= readOff);
+ Assert (pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
-
+ Assert (pageHeaderSize + len <= state->readLen);
memcpy(buffer, (char *) contdata, len);
buffer += len;
gotlen += len;
@@ -488,9 +493,15 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -533,109 +544,139 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. header_inclusive indicates that
+ * reqLen is calculated including page header length.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or found
+ * error. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall call this function
+ * again after filling the buffer at least with that portion of data and set
+ * state->readLen to the length of actually loaded data.
+ *
+ * If header_inclusive is false, corrects reqLen internally by adding the
+ * actual page header length and may request caller for new data.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool header_inclusive)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
+ uint32 addLen = 0;
- Assert((pageptr % XLOG_BLCKSZ) == 0);
+ /* Some data is loaded, but page header is not verified yet. */
+ if (!state->page_verified &&
+ !XLogRecPtrIsInvalid(state->readPagePtr) && state->readLen >= 0)
+ {
+ uint32 pageHeaderSize;
- XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
- targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ /* just loaded new data so needs to verify page header */
- /* check whether we have all the requested data already */
- if (targetSegNo == state->seg.ws_segno &&
- targetPageOff == state->segoff && reqLen <= state->readLen)
- return state->readLen;
+ /* The caller must have loaded at least page header */
+ Assert (state->readLen >= SizeOfXLogShortPHD);
- /*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
- * Whenever switching to a new WAL segment, we read the first page of the
- * file and validate its header, even if that's not where the target
- * record is. This is so that we can check the additional identification
- * info that is present in the first page's "long" header.
- */
- if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
- {
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length against the actual header length.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Request more data if we don't have the full header. */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /* Now that we know we have the full header, validate it. */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* That's bad. Force reading the page again. */
+ XLogReaderInvalReadState(state);
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
+ return false;
+ }
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->seg.ws_segno,
+ state->segcxt.ws_segsize);
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * The loaded page may not be the one caller is supposing to read when we
+ * are verifying the first page of new segment. In that case, skip further
+ * verification and immediately load the target page.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
+ /*
+ * calculate additional length for page header keeping the total
+ * length within the block size.
+ */
+ if (!header_inclusive)
+ {
+ uint32 pageHeaderSize =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
- Assert(readLen >= reqLen);
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
- hdr = (XLogPageHeader) state->readBuf;
+ Assert(addLen >= 0);
+ }
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf);
- if (readLen < 0)
- goto err;
+ /* Return if we already have it. */
+ if (reqLen + addLen <= state->readLen)
+ return false;
}
+ /* Data is not in our buffer, request the caller for it. */
+ XLByteToSeg(pageptr, targetSegNo, state->segcxt.ws_segsize);
+ targetPageOff = XLogSegmentOffset(pageptr, state->segcxt.ws_segsize);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
/*
- * Now that we know we have the full header, validate it.
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
*/
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->seg.ws_segno = targetSegNo;
- state->segoff = targetPageOff;
- state->readLen = readLen;
+ state->page_verified = false;
- return readLen;
+ /*
+ * Whenever switching to a new WAL segment, we read the first page of the
+ * file and validate its header, even if that's not where the target
+ * record is. This is so that we can check the additional identification
+ * info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
+ */
+ if (targetSegNo != state->seg.ws_segno && targetPageOff != 0)
+ {
+ /*
+ * Then we'll see that the targetSegNo now matches the ws_segno, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
+ }
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ /*
+ * Request the caller to load the page. We need at least a short page
+ * header so that we can validate it.
+ */
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
@@ -644,9 +685,7 @@ err:
static void
XLogReaderInvalReadState(XLogReaderState *state)
{
- state->seg.ws_segno = 0;
- state->segoff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -924,7 +963,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -932,7 +970,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -940,19 +978,23 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
/* scroll back to page boundary */
targetPagePtr = tmpRecPtr - targetRecOff;
- /* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff,
+ targetRecOff != 0))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
pageHeaderSize = XLogPageHeaderSize(header);
- /* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
- goto err;
+ /* we should have read the page header */
+ Assert (state->readLen >= pageHeaderSize);
/* skip over potential continuation data */
if (header->xlp_info & XLP_FIRST_IS_CONTRECORD)
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 446760ed6e..060fbc0c4c 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -680,8 +680,8 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum,
void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
- const XLogRecPtr lastReadPage = (state->seg.ws_segno *
- state->segcxt.ws_segsize + state->segoff);
+ const XLogRecPtr lastReadPage = state->seg.ws_segno *
+ state->segcxt.ws_segsize + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -813,7 +813,7 @@ wal_segment_open(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -915,7 +915,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -933,7 +934,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
WALReadRaiseError(&errinfo);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index aa2106b152..4b0e8ea773 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -106,7 +106,7 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index ac9209747a..e2d64cc2e5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -762,7 +762,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page)
{
@@ -782,7 +782,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -812,7 +815,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
XLByteToSeg(targetPagePtr, segno, sendCxt->ws_segsize);
CheckXLogRemoved(segno, sendSeg->ws_tli);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 1d03375a3e..795e84ff9f 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -44,7 +44,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
@@ -228,7 +228,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
{
@@ -283,7 +283,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -296,7 +297,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -309,13 +311,15 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index 30a5851d87..9eece44626 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -324,9 +324,9 @@ WALDumpOpenSegment(XLogSegNo nextSegNo, WALSegmentContext *segcxt,
/*
* XLogReader read_page callback
*/
-static int
-WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
- XLogRecPtr targetPtr, char *readBuff)
+static bool
+XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
+ XLogRecPtr targetPtr, char *readBuff)
{
XLogDumpPrivate *private = state->private_data;
int count = XLOG_BLCKSZ;
@@ -341,7 +341,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
@@ -366,7 +367,8 @@ WALDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
(Size) errinfo.wre_req);
}
- return count;
+ state->readLen = count;
+ return true;
}
/*
@@ -1027,7 +1029,7 @@ main(int argc, char **argv)
/* done with argument parsing, do the actual work */
/* we have everything we need, start reading */
- xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, WALDumpReadPage,
+ xlogreader_state = XLogReaderAllocate(WalSegSz, waldir, XLogDumpReadPage,
&private);
if (!xlogreader_state)
fatal_error("out of memory");
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index 0193611b7f..d990432ffe 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -49,7 +49,7 @@ typedef struct WALSegmentContext
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -131,6 +131,20 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
+ bool page_verified; /* is the page on the buffer verified? */
+
/* ----------------------------------------
* Decoded representation of current record
@@ -157,13 +171,6 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
/* last read XLOG position for data currently in readBuf */
WALSegmentContext segcxt;
WALOpenSegment seg;
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 0572b24192..0867ef0599 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index 012096f183..54291221c3 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page);
--
2.23.0
----Next_Part(Wed_Nov_27_12_09_23_2019_240)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v11-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v5 1/3] Move callback-call from ReadPageInternal to XLogReadRecord.
@ 2019-09-05 11:21 Kyotaro Horiguchi <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Kyotaro Horiguchi @ 2019-09-05 11:21 UTC (permalink / raw)
WAL reader facility used to call given page-reader callback function
in the ReadPageInternal, which is two levels below from
ReadRecord. That makes things a bit complex, but furthermore we are
going to have additional callbacks for encryption, compression or
something like that works before or after reading pages. Just adding
them as new callback makes things messier. If the caller of the
current ReadRecord could call page fetching function directly, things
would get quite easier.
As the first step of that change, this patch moves the place where
that callbacks are called by 1 level above
ReadPageInternal. XLogPageRead uses a loop over new function
XLogNeedData and the callback directly instead of ReadPageInternal.
---
src/backend/access/transam/xlog.c | 15 +-
src/backend/access/transam/xlogreader.c | 355 ++++++++++++++++---------
src/backend/access/transam/xlogutils.c | 10 +-
src/backend/replication/logical/logicalfuncs.c | 4 +-
src/backend/replication/walsender.c | 10 +-
src/bin/pg_rewind/parsexlog.c | 17 +-
src/bin/pg_waldump/pg_waldump.c | 8 +-
src/include/access/xlogreader.h | 30 ++-
src/include/access/xlogutils.h | 2 +-
src/include/replication/logicalfuncs.h | 2 +-
10 files changed, 292 insertions(+), 161 deletions(-)
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e651a841bb..acd4ed89c4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -884,7 +884,7 @@ static bool InstallXLogFileSegment(XLogSegNo *segno, char *tmppath,
static int XLogFileRead(XLogSegNo segno, int emode, TimeLineID tli,
int source, bool notfoundOk);
static int XLogFileReadAnyTLI(XLogSegNo segno, int emode, int source);
-static int XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
+static bool XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *readTLI);
static bool WaitForWALToBecomeAvailable(XLogRecPtr RecPtr, bool randAccess,
@@ -11521,7 +11521,7 @@ CancelBackup(void)
* XLogPageRead() to try fetching the record from another source, or to
* sleep and retry.
*/
-static int
+static bool
XLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *readBuf, TimeLineID *readTLI)
{
@@ -11580,7 +11580,8 @@ retry:
readLen = 0;
readSource = 0;
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -11675,7 +11676,8 @@ retry:
goto next_record_is_invalid;
}
- return readLen;
+ xlogreader->readLen = readLen;
+ return true;
next_record_is_invalid:
lastSourceFailed = true;
@@ -11689,8 +11691,9 @@ next_record_is_invalid:
/* In standby-mode, keep trying */
if (StandbyMode)
goto retry;
- else
- return -1;
+
+ xlogreader->readLen = -1;
+ return false;
}
/*
diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c
index c6faf48d24..1d8d470158 100644
--- a/src/backend/access/transam/xlogreader.c
+++ b/src/backend/access/transam/xlogreader.c
@@ -36,8 +36,8 @@ static bool ValidXLogRecordHeader(XLogReaderState *state, XLogRecPtr RecPtr,
XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess);
static bool ValidXLogRecord(XLogReaderState *state, XLogRecord *record,
XLogRecPtr recptr);
-static int ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr,
- int reqLen);
+static bool XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr,
+ int reqLen, bool includes_page_header);
static void report_invalid_record(XLogReaderState *state, const char *fmt,...) pg_attribute_printf(2, 3);
static void ResetDecoder(XLogReaderState *state);
@@ -100,7 +100,7 @@ XLogReaderAllocate(int wal_segment_size, XLogPageReadCB pagereadfunc,
/* system_identifier initialized to zeroes above */
state->private_data = private_data;
/* ReadRecPtr and EndRecPtr initialized to zeroes above */
- /* readSegNo, readOff, readLen, readPageTLI initialized to zeroes above */
+ /* readSegNo, readLen, readPageTLI initialized to zeroes above */
state->errormsg_buf = palloc_extended(MAX_ERRORMSG_LEN + 1,
MCXT_ALLOC_NO_OOM);
if (!state->errormsg_buf)
@@ -224,7 +224,6 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
uint32 targetRecOff;
uint32 pageHeaderSize;
bool gotheader;
- int readOff;
/*
* randAccess indicates whether to verify the previous-record pointer of
@@ -276,15 +275,21 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
* byte to cover the whole record header, or at least the part of it that
* fits on the same page.
*/
- readOff = ReadPageInternal(state,
- targetPagePtr,
- Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ),
+ true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/*
- * ReadPageInternal always returns at least the page header, so we can
- * examine it now.
+ * We have loaded at least the page header, so we can examine it now.
*/
pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
if (targetRecOff == 0)
@@ -310,8 +315,8 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
goto err;
}
- /* ReadPageInternal has verified the page header */
- Assert(pageHeaderSize <= readOff);
+ /* XLogNeedData has verified the page header */
+ Assert(pageHeaderSize <= state->readLen);
/*
* Read the record length.
@@ -388,14 +393,21 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
targetPagePtr += XLOG_BLCKSZ;
/* Wait for the next page to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(total_len - gotlen + SizeOfXLogShortPHD,
- XLOG_BLCKSZ));
+ while (XLogNeedData(state, targetPagePtr,
+ Min(total_len - gotlen + SizeOfXLogShortPHD,
+ XLOG_BLCKSZ),
+ false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
- if (readOff < 0)
+ if (!state->page_verified)
goto err;
- Assert(SizeOfXLogShortPHD <= readOff);
+ Assert(SizeOfXLogShortPHD <= state->readLen);
/* Check that the continuation on next page looks valid */
pageHeader = (XLogPageHeader) state->readBuf;
@@ -424,20 +436,38 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
/* Append the continuation from this page to the buffer */
pageHeaderSize = XLogPageHeaderSize(pageHeader);
- if (readOff < pageHeaderSize)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize);
+ if (state->readLen < pageHeaderSize)
+ {
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize,
+ false))
+ {
+ if (!state->read_page(state,
+ state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+ }
- Assert(pageHeaderSize <= readOff);
+ Assert(pageHeaderSize <= state->readLen);
contdata = (char *) state->readBuf + pageHeaderSize;
len = XLOG_BLCKSZ - pageHeaderSize;
if (pageHeader->xlp_rem_len < len)
len = pageHeader->xlp_rem_len;
- if (readOff < pageHeaderSize + len)
- readOff = ReadPageInternal(state, targetPagePtr,
- pageHeaderSize + len);
+ if (state->readLen < pageHeaderSize + len)
+ {
+ if (XLogNeedData(state, targetPagePtr, pageHeaderSize + len,
+ true))
+ {
+ if (!state->read_page(state,
+ state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+ }
memcpy(buffer, (char *) contdata, len);
buffer += len;
@@ -468,9 +498,16 @@ XLogReadRecord(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg)
else
{
/* Wait for the record data to become available */
- readOff = ReadPageInternal(state, targetPagePtr,
- Min(targetRecOff + total_len, XLOG_BLCKSZ));
- if (readOff < 0)
+ while (XLogNeedData(state, targetPagePtr,
+ Min(targetRecOff + total_len, XLOG_BLCKSZ), true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* Record does not cross a page boundary */
@@ -504,7 +541,7 @@ err:
* Invalidate the read state. We might read from a different source after
* failure.
*/
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
if (state->errormsg_buf[0] != '\0')
*errormsg = state->errormsg_buf;
@@ -513,120 +550,181 @@ err:
}
/*
- * Read a single xlog page including at least [pageptr, reqLen] of valid data
- * via the read_page() callback.
+ * Checks that an xlog page loaded in state->readBuf is including at least
+ * [pageptr, reqLen] and the page is valid. includes_page_header indicates that
+ * the requested region contains page header.
*
- * Returns -1 if the required page cannot be read for some reason; errormsg_buf
- * is set in that case (unless the error occurs in the read_page callback).
+ * Returns false if the buffer already contains the requested data, or error is
+ * found. state->page_verified is set to true for the former and false for the
+ * latter.
*
- * We fetch the page from a reader-local cache if we know we have the required
- * data and if there hasn't been any error since caching the data.
+ * Otherwise returns true and requests data loaded onto state->readBuf by
+ * state->readPagePtr and state->readLen. The caller shall fill the buffer at
+ * least with that portion of data and set state->readLen to the actual length
+ * of loaded data before the next call to this function.
+ *
+ * If reqLen does not contain page header, includes_page_header should be
+ * true. This function internally adds page header to reqLen.
*/
-static int
-ReadPageInternal(XLogReaderState *state, XLogRecPtr pageptr, int reqLen)
+static bool
+XLogNeedData(XLogReaderState *state, XLogRecPtr pageptr, int reqLen,
+ bool includes_page_header)
{
- int readLen;
uint32 targetPageOff;
XLogSegNo targetSegNo;
- XLogPageHeader hdr;
-
- Assert((pageptr % XLOG_BLCKSZ) == 0);
-
- XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
- targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ uint32 addLen = 0;
/* check whether we have all the requested data already */
- if (targetSegNo == state->readSegNo && targetPageOff == state->readOff &&
- reqLen <= state->readLen)
- return state->readLen;
+ if (state->page_verified && pageptr == state->readPagePtr)
+ {
+ if (includes_page_header)
+ {
+ /*
+ * Include page header length in request, but the total shoudn't
+ * exceed the block size.
+ */
+ uint32 headerLen =
+ XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ if (reqLen + headerLen <= XLOG_BLCKSZ)
+ addLen = headerLen;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+ }
+
+ if (reqLen + addLen <= state->readLen)
+ return false;
+ }
+
+ /* Haven't loaded any data yet? Then request it. */
+ if (XLogRecPtrIsInvalid(state->readPagePtr) || state->readLen < 0)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen, SizeOfXLogShortPHD);
+ state->page_verified = false;
+ return true;
+ }
+
+ if (!state->page_verified)
+ {
+ uint32 pageHeaderSize;
+ uint32 addLen = 0;
+
+ /* just loaded new data so needs to verify page header */
+
+ /* The caller must have loaded at least page header */
+ Assert(state->readLen >= SizeOfXLogShortPHD);
+
+ /*
+ * We have enough data to check the header length. Recheck the loaded
+ * length if it is a long header if any.
+ */
+ pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf);
+
+ /*
+ * If we have not loaded a page so far, readLen is zero, which is
+ * shorter than pageHeaderSize here.
+ */
+ if (state->readLen < pageHeaderSize)
+ {
+ state->readPagePtr = pageptr;
+ state->readLen = pageHeaderSize;
+ return true;
+ }
+
+ /*
+ * Now that we know we have the full header, validate it.
+ */
+ if (!XLogReaderValidatePageHeader(state, state->readPagePtr,
+ (char *) state->readBuf))
+ {
+ /* force reading the page again. */
+ XLogReaderDiscardReadingPage(state);
+
+ return false;
+ }
+
+ state->page_verified = true;
+
+ XLByteToSeg(state->readPagePtr, state->readSegNo,
+ state->wal_segment_size);
+
+ /*
+ * calculate additional length for page header so that the total length
+ * doesn't exceed the block size.
+ */
+ if (includes_page_header)
+ {
+ addLen = pageHeaderSize;
+ if (reqLen + pageHeaderSize <= XLOG_BLCKSZ)
+ addLen = pageHeaderSize;
+ else
+ addLen = XLOG_BLCKSZ - reqLen;
+
+ Assert(addLen >= 0);
+ }
+
+ /* Usually we have requested data loaded in buffer here. */
+ if (pageptr == state->readPagePtr && reqLen + addLen <= state->readLen)
+ return false;
+
+ /*
+ * In the case the page is requested for the first record in the page,
+ * the previous request have been made not counting page header
+ * length. Request again for the same page with the length known to be
+ * needed. Otherwise we don't know the header length of the new page.
+ */
+ if (pageptr != state->readPagePtr)
+ addLen = 0;
+ }
+
+ /* Data is not in our buffer, make a new load request to the caller. */
+ XLByteToSeg(pageptr, targetSegNo, state->wal_segment_size);
+ targetPageOff = XLogSegmentOffset(pageptr, state->wal_segment_size);
+ Assert((pageptr % XLOG_BLCKSZ) == 0);
+
+ /*
+ * Every time we request to load new data of a page to the caller, even if
+ * we looked at a part of it before, we need to do verification on the next
+ * invocation as the caller might now be rereading data from a different
+ * source.
+ */
+ state->page_verified = false;
/*
- * Data is not in our buffer.
- *
- * Every time we actually read the page, even if we looked at parts of it
- * before, we need to do verification as the read_page callback might now
- * be rereading data from a different source.
- *
* Whenever switching to a new WAL segment, we read the first page of the
* file and validate its header, even if that's not where the target
* record is. This is so that we can check the additional identification
* info that is present in the first page's "long" header.
+ * Don't do this if the caller requested the first page in the segment.
*/
if (targetSegNo != state->readSegNo && targetPageOff != 0)
{
- XLogRecPtr targetSegmentPtr = pageptr - targetPageOff;
-
- readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ,
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- /* we can be sure to have enough WAL available, we scrolled back */
- Assert(readLen == XLOG_BLCKSZ);
-
- if (!XLogReaderValidatePageHeader(state, targetSegmentPtr,
- state->readBuf))
- goto err;
+ /*
+ * Then we'll see that the targetSegNo now matches the readSegNo, and
+ * will not come back here, but will request the actual target page.
+ */
+ state->readPagePtr = pageptr - targetPageOff;
+ state->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
- * First, read the requested data length, but at least a short page header
- * so that we can validate it.
+ * Request the caller to load the requested page. We need at least a short
+ * page header so that we can validate it.
*/
- readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
-
- Assert(readLen <= XLOG_BLCKSZ);
-
- /* Do we have enough data to check the header length? */
- if (readLen <= SizeOfXLogShortPHD)
- goto err;
-
- Assert(readLen >= reqLen);
-
- hdr = (XLogPageHeader) state->readBuf;
-
- /* still not enough */
- if (readLen < XLogPageHeaderSize(hdr))
- {
- readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr),
- state->currRecPtr,
- state->readBuf, &state->readPageTLI);
- if (readLen < 0)
- goto err;
- }
-
- /*
- * Now that we know we have the full header, validate it.
- */
- if (!XLogReaderValidatePageHeader(state, pageptr, (char *) hdr))
- goto err;
-
- /* update read state information */
- state->readSegNo = targetSegNo;
- state->readOff = targetPageOff;
- state->readLen = readLen;
-
- return readLen;
-
-err:
- XLogReaderInvalReadState(state);
- return -1;
+ state->readPagePtr = pageptr;
+ state->readLen = Max(reqLen + addLen, SizeOfXLogShortPHD);
+ return true;
}
/*
- * Invalidate the xlogreader's read state to force a re-read.
+ * Invalidate current reading page buffer
*/
void
-XLogReaderInvalReadState(XLogReaderState *state)
+XLogReaderDiscardReadingPage(XLogReaderState *state)
{
- state->readSegNo = 0;
- state->readOff = 0;
- state->readLen = 0;
+ state->readPagePtr = InvalidXLogRecPtr;
}
/*
@@ -904,7 +1002,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
XLogRecPtr targetPagePtr;
int targetRecOff;
uint32 pageHeaderSize;
- int readLen;
/*
* Compute targetRecOff. It should typically be equal or greater than
@@ -912,7 +1009,7 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
* that, except when caller has explicitly specified the offset that
* falls somewhere there or when we are skipping multi-page
* continuation record. It doesn't matter though because
- * ReadPageInternal() is prepared to handle that and will read at
+ * CheckPage() is prepared to handle that and will read at
* least short page-header worth of data
*/
targetRecOff = tmpRecPtr % XLOG_BLCKSZ;
@@ -921,8 +1018,15 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
targetPagePtr = tmpRecPtr - targetRecOff;
/* Read the page containing the record */
- readLen = ReadPageInternal(state, targetPagePtr, targetRecOff);
- if (readLen < 0)
+ while(XLogNeedData(state, targetPagePtr, targetRecOff, true))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
header = (XLogPageHeader) state->readBuf;
@@ -930,8 +1034,15 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
pageHeaderSize = XLogPageHeaderSize(header);
/* make sure we have enough data for the page header */
- readLen = ReadPageInternal(state, targetPagePtr, pageHeaderSize);
- if (readLen < 0)
+ while (XLogNeedData(state, targetPagePtr, pageHeaderSize, false))
+ {
+ if (!state->read_page(state, state->readPagePtr, state->readLen,
+ state->ReadRecPtr, state->readBuf,
+ &state->readPageTLI))
+ break;
+ }
+
+ if (!state->page_verified)
goto err;
/* skip over potential continuation data */
@@ -989,7 +1100,7 @@ out:
/* Reset state to what we had before finding the record */
state->ReadRecPtr = saved_state.ReadRecPtr;
state->EndRecPtr = saved_state.EndRecPtr;
- XLogReaderInvalReadState(state);
+ XLogReaderDiscardReadingPage(state);
return found;
}
diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c
index 1fc39333f1..dad9074b9f 100644
--- a/src/backend/access/transam/xlogutils.c
+++ b/src/backend/access/transam/xlogutils.c
@@ -803,7 +803,7 @@ void
XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wantLength)
{
const XLogRecPtr lastReadPage = state->readSegNo *
- state->wal_segment_size + state->readOff;
+ state->wal_segment_size + state->readLen;
Assert(wantPage != InvalidXLogRecPtr && wantPage % XLOG_BLCKSZ == 0);
Assert(wantLength <= XLOG_BLCKSZ);
@@ -907,7 +907,7 @@ XLogReadDetermineTimeline(XLogReaderState *state, XLogRecPtr wantPage, uint32 wa
* exists for normal backends, so we have to do a check/sleep/repeat style of
* loop for now.
*/
-int
+bool
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI)
@@ -1009,7 +1009,8 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
else if (targetPagePtr + reqLen > read_upto)
{
/* not enough data there */
- return -1;
+ state->readLen = -1;
+ return false;
}
else
{
@@ -1026,5 +1027,6 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
XLOG_BLCKSZ);
/* number of valid bytes in the buffer */
- return count;
+ state->readLen = count;
+ return true;
}
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index d974400d6e..7210a940bd 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -114,12 +114,12 @@ check_permissions(void)
(errmsg("must be superuser or replication role to use replication slots"))));
}
-int
+bool
logical_read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
return read_local_xlog_page(state, targetPagePtr, reqLen,
- targetRecPtr, cur_page, pageTLI);
+ targetRecPtr, cur_page, pageTLI);
}
/*
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 23870a25a5..cc35e2a04d 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -761,7 +761,7 @@ StartReplication(StartReplicationCmd *cmd)
* which has to do a plain sleep/busy loop, because the walsender's latch gets
* set every time WAL is flushed.
*/
-static int
+static bool
logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI)
{
@@ -779,7 +779,10 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* fail if not (implies we are going to shut down) */
if (flushptr < targetPagePtr + reqLen)
- return -1;
+ {
+ state->readLen = -1;
+ return false;
+ }
if (targetPagePtr + XLOG_BLCKSZ <= flushptr)
count = XLOG_BLCKSZ; /* more than one block available */
@@ -789,7 +792,8 @@ logical_read_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int req
/* now actually read the data, we know it's there */
XLogRead(cur_page, targetPagePtr, XLOG_BLCKSZ);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 63c3879ead..4df53964e4 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -47,7 +47,7 @@ typedef struct XLogPageReadPrivate
int tliIndex;
} XLogPageReadPrivate;
-static int SimpleXLogPageRead(XLogReaderState *xlogreader,
+static bool SimpleXLogPageRead(XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
@@ -235,7 +235,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
/* XLogReader callback function, to read a WAL page */
-static int
+static bool
SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI)
@@ -290,7 +290,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
}
@@ -303,7 +304,8 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
{
pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
@@ -316,13 +318,16 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
pg_log_error("could not read file \"%s\": read %d of %zu",
xlogfpath, r, (Size) XLOG_BLCKSZ);
- return -1;
+ xlogreader->readLen = -1;
+ return false;
}
Assert(targetSegNo == xlogreadsegno);
*pageTLI = targetHistory[private->tliIndex].tli;
- return XLOG_BLCKSZ;
+
+ xlogreader->readLen = XLOG_BLCKSZ;
+ return true;
}
/*
diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c
index b95d467805..96d1f36ebc 100644
--- a/src/bin/pg_waldump/pg_waldump.c
+++ b/src/bin/pg_waldump/pg_waldump.c
@@ -421,7 +421,7 @@ XLogDumpXLogRead(const char *directory, TimeLineID timeline_id,
/*
* XLogReader read_page callback
*/
-static int
+static bool
XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI)
{
@@ -437,14 +437,16 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
else
{
private->endptr_reached = true;
- return -1;
+ state->readLen = -1;
+ return false;
}
}
XLogDumpXLogRead(private->inpath, private->timeline, targetPagePtr,
readBuff, count);
- return count;
+ state->readLen = count;
+ return true;
}
/*
diff --git a/src/include/access/xlogreader.h b/src/include/access/xlogreader.h
index aa9bc63725..030f56802b 100644
--- a/src/include/access/xlogreader.h
+++ b/src/include/access/xlogreader.h
@@ -34,7 +34,7 @@
typedef struct XLogReaderState XLogReaderState;
/* Function type definition for the read_page callback */
-typedef int (*XLogPageReadCB) (XLogReaderState *xlogreader,
+typedef bool (*XLogPageReadCB) (XLogReaderState *xlogreader,
XLogRecPtr targetPagePtr,
int reqLen,
XLogRecPtr targetRecPtr,
@@ -123,6 +123,18 @@ struct XLogReaderState
XLogRecPtr ReadRecPtr; /* start of last record read */
XLogRecPtr EndRecPtr; /* end+1 of last record read */
+ /* ----------------------------------------
+ * Communication with page reader
+ * readBuf is XLOG_BLCKSZ bytes, valid up to at least readLen bytes.
+ * ----------------------------------------
+ */
+ /* variables to communicate with page reader */
+ XLogRecPtr readPagePtr; /* page pointer to read */
+ int32 readLen; /* bytes requested to reader, or actual bytes
+ * read by reader, which must be larger than
+ * the request, or -1 on error */
+ TimeLineID readPageTLI; /* TLI for data currently in readBuf */
+ char *readBuf; /* buffer to store data */
/* ----------------------------------------
* Decoded representation of current record
@@ -149,17 +161,9 @@ struct XLogReaderState
* ----------------------------------------
*/
- /*
- * Buffer for currently read page (XLOG_BLCKSZ bytes, valid up to at least
- * readLen bytes)
- */
- char *readBuf;
- uint32 readLen;
-
- /* last read segment, segment offset, TLI for data currently in readBuf */
+ /* last read segment and segment offset for data currently in readBuf */
+ bool page_verified;
XLogSegNo readSegNo;
- uint32 readOff;
- TimeLineID readPageTLI;
/*
* beginning of prior page read, and its TLI. Doesn't necessarily
@@ -216,8 +220,8 @@ extern struct XLogRecord *XLogReadRecord(XLogReaderState *state,
extern bool XLogReaderValidatePageHeader(XLogReaderState *state,
XLogRecPtr recptr, char *phdr);
-/* Invalidate read state */
-extern void XLogReaderInvalReadState(XLogReaderState *state);
+/* Discard bufferd page */
+extern void XLogReaderDiscardReadingPage(XLogReaderState *state);
#ifdef FRONTEND
extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h
index 4105b59904..0842af9f95 100644
--- a/src/include/access/xlogutils.h
+++ b/src/include/access/xlogutils.h
@@ -47,7 +47,7 @@ extern Buffer XLogReadBufferExtended(RelFileNode rnode, ForkNumber forknum,
extern Relation CreateFakeRelcacheEntry(RelFileNode rnode);
extern void FreeFakeRelcacheEntry(Relation fakerel);
-extern int read_local_xlog_page(XLogReaderState *state,
+extern bool read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page,
TimeLineID *pageTLI);
diff --git a/src/include/replication/logicalfuncs.h b/src/include/replication/logicalfuncs.h
index a9c178a9e6..8e52b1f4aa 100644
--- a/src/include/replication/logicalfuncs.h
+++ b/src/include/replication/logicalfuncs.h
@@ -11,7 +11,7 @@
#include "replication/logical.h"
-extern int logical_read_local_xlog_page(XLogReaderState *state,
+extern bool logical_read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, TimeLineID *pageTLI);
--
2.16.3
----Next_Part(Fri_Sep_06_16_33_18_2019_198)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v5-0002-Move-page-reader-out-of-XLogReadRecord.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress in server logs
@ 2022-01-21 05:37 Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-01-21 05:37 UTC (permalink / raw)
To: Bruce Momjian <[email protected]>; +Cc: Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; Bharath Rupireddy <[email protected]>; PostgreSQL Hackers <[email protected]>
> I think the right choice to solve the *general* problem is the
> mentioned pg_stat_progress_checkpoints.
>
> We may want to *additionally* have the ability to log the progress
> specifically for the special cases when we're not able to use that
> view. And in those case, we can perhaps just use the existing
> log_startup_progress_interval parameter for this as well -- at least
> for the startup checkpoint.
+1
> We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
>
> Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
I agree to provide above mentioned information as part of showing the
progress of current checkpoint operation. I am currently looking into
the code to know if any other information can be added.
Thanks & Regards,
Nitin Jadhav
On Thu, Jan 6, 2022 at 5:12 AM Bruce Momjian <[email protected]> wrote:
>
> On Wed, Dec 29, 2021 at 10:40:59AM -0500, Tom Lane wrote:
> > Magnus Hagander <[email protected]> writes:
> > >> Therefore, reporting the checkpoint progress in the server logs, much
> > >> like [1], seems to be the best way IMO.
> >
> > > I find progress reporting in the logfile to generally be a terrible
> > > way of doing things, and the fact that we do it for the startup
> > > process is/should be only because we have no other choice, not because
> > > it's the right choice.
> >
> > I'm already pretty seriously unhappy about the log-spamming effects of
> > 64da07c41 (default to log_checkpoints=on), and am willing to lay a side
> > bet that that gets reverted after we have some field experience with it.
> > This proposal seems far worse from that standpoint. Keep in mind that
> > our out-of-the-box logging configuration still doesn't have any log
> > rotation ability, which means that the noisier the server is in normal
> > operation, the sooner you fill your disk.
>
> I think we are looking at three potential observable behaviors people
> might care about:
>
> * the current activity/progress of checkpoints
> * the historical reporting of checkpoint completion, mixed in with other
> log messages for later analysis
> * the aggregate behavior of checkpoint operation
>
> I think it is clear that checkpoint progress activity isn't useful for
> the server logs because that information has little historical value,
> but does fit for a progress view. As Tom already expressed, we will
> have to wait to see if non-progress checkpoint information in the logs
> has sufficient historical value.
>
> --
> Bruce Momjian <[email protected]> https://momjian.us
> EDB https://enterprisedb.com
>
> If only the physical world exists, free will is an illusion.
>
>
>
^ permalink raw reply [nested|flat] 85+ messages in thread
* Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
@ 2022-01-28 06:54 ` Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Bharath Rupireddy @ 2022-01-28 06:54 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Jan 21, 2022 at 11:07 AM Nitin Jadhav
<[email protected]> wrote:
>
> > I think the right choice to solve the *general* problem is the
> > mentioned pg_stat_progress_checkpoints.
> >
> > We may want to *additionally* have the ability to log the progress
> > specifically for the special cases when we're not able to use that
> > view. And in those case, we can perhaps just use the existing
> > log_startup_progress_interval parameter for this as well -- at least
> > for the startup checkpoint.
>
> +1
>
> > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> >
> > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
>
> I agree to provide above mentioned information as part of showing the
> progress of current checkpoint operation. I am currently looking into
> the code to know if any other information can be added.
As suggested in the other thread by Julien, I'm changing the subject
of this thread to reflect the discussion.
Regards,
Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
@ 2022-02-10 06:52 ` Nitin Jadhav <[email protected]>
2022-02-15 12:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-22 14:40 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
0 siblings, 3 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-10 06:52 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> >
> > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
>
> I agree to provide above mentioned information as part of showing the
> progress of current checkpoint operation. I am currently looking into
> the code to know if any other information can be added.
Here is the initial patch to show the progress of checkpoint through
pg_stat_progress_checkpoint view. Please find the attachment.
The information added to this view are pid - process ID of a
CHECKPOINTER process, kind - kind of checkpoint indicates the reason
for checkpoint (values can be wal, time or force), phase - indicates
the current phase of checkpoint operation, total_buffer_writes - total
number of buffers to be written, buffers_processed - number of buffers
processed, buffers_written - number of buffers written,
total_file_syncs - total number of files to be synced, files_synced -
number of files synced.
There are many operations happen as part of checkpoint. For each of
the operation I am updating the phase field of
pg_stat_progress_checkpoint view. The values supported for this field
are initializing, checkpointing replication slots, checkpointing
snapshots, checkpointing logical rewrite mappings, checkpointing CLOG
pages, checkpointing CommitTs pages, checkpointing SUBTRANS pages,
checkpointing MULTIXACT pages, checkpointing SLRU pages, checkpointing
buffers, performing sync requests, performing two phase checkpoint,
recycling old XLOG files and Finalizing. In case of checkpointing
buffers phase, the fields total_buffer_writes, buffers_processed and
buffers_written shows the detailed progress of writing buffers. In
case of performing sync requests phase, the fields total_file_syncs
and files_synced shows the detailed progress of syncing files. In
other phases, only the phase field is getting updated and it is
difficult to show the progress because we do not get the total number
of files count without traversing the directory. It is not worth to
calculate that as it affects the performance of the checkpoint. I also
gave a thought to just mention the number of files processed, but this
wont give a meaningful progress information (It can be treated as
statistics). Hence just updating the phase field in those scenarios.
Apart from above fields, I am planning to add few more fields to the
view in the next patch. That is, process ID of the backend process
which triggered a CHECKPOINT command, checkpoint start location, filed
to indicate whether it is a checkpoint or restartpoint and elapsed
time of the checkpoint operation. Please share your thoughts. I would
be happy to add any other information that contributes to showing the
progress of checkpoint.
As per the discussion in this thread, there should be some mechanism
to show the progress of checkpoint during shutdown and end-of-recovery
cases as we cannot access pg_stat_progress_checkpoint in those cases.
I am working on this to use log_startup_progress_interval mechanism to
log the progress in the server logs.
Kindly review the patch and share your thoughts.
On Fri, Jan 28, 2022 at 12:24 PM Bharath Rupireddy
<[email protected]> wrote:
>
> On Fri, Jan 21, 2022 at 11:07 AM Nitin Jadhav
> <[email protected]> wrote:
> >
> > > I think the right choice to solve the *general* problem is the
> > > mentioned pg_stat_progress_checkpoints.
> > >
> > > We may want to *additionally* have the ability to log the progress
> > > specifically for the special cases when we're not able to use that
> > > view. And in those case, we can perhaps just use the existing
> > > log_startup_progress_interval parameter for this as well -- at least
> > > for the startup checkpoint.
> >
> > +1
> >
> > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > >
> > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> >
> > I agree to provide above mentioned information as part of showing the
> > progress of current checkpoint operation. I am currently looking into
> > the code to know if any other information can be added.
>
> As suggested in the other thread by Julien, I'm changing the subject
> of this thread to reflect the discussion.
>
> Regards,
> Bharath Rupireddy.
Attachments:
[application/octet-stream] 0001-pg_stat_progress_checkpoint-view.patch (25.3K, ../../CAMm1aWY_wThAxmhC_7mkFsYHcNP2_JQLAKJEoY5EcHBi0Q42QQ@mail.gmail.com/2-0001-pg_stat_progress_checkpoint-view.patch)
download | inline diff:
From 5b68df74d68922685dff4eb665a6431c5046f848 Mon Sep 17 00:00:00 2001
From: Nitin Jadhav <[email protected]>
Date: Wed, 9 Feb 2022 12:00:11 +0000
Subject: [PATCH] pg_stat_progress_checkpoint view
---
doc/src/sgml/monitoring.sgml | 263 +++++++++++++++++++++++++++
src/backend/access/transam/xlog.c | 123 ++++++++++++-
src/backend/catalog/system_views.sql | 29 +++
src/backend/storage/buffer/bufmgr.c | 10 +
src/backend/storage/sync/sync.c | 10 +-
src/backend/utils/adt/pgstatfuncs.c | 2 +
src/include/access/xlog.h | 4 +
src/include/commands/progress.h | 31 ++++
src/include/storage/sync.h | 2 +-
src/include/utils/backend_progress.h | 3 +-
10 files changed, 473 insertions(+), 4 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 62f2a3332b..a6893d4543 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -401,6 +401,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of <command>CHECKPOINT</command> operation.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -6886,6 +6893,262 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>CHECKPOINT Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of checkpoint operation. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of a CHECKPOINTER process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>kinds</structfield> <type>text</type>
+ </para>
+ <para>
+ Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_buffer_writes</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal> total_buffer_writes </literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>total_buffer_writes</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>total_file_syncs</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>total_file_syncs</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-kinds">
+ <title>CHECKPOINT kinds</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Kinds</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint operation is requested due to XLOG filling.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint operation is requested due to timeout.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint operation is forced even if no XLOG activity has occurred
+ since the last one.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>CHECKPOINT phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The CHECKPOINTER process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently flushing all the replication slots to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing snapshots</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently removing all the serialized
+ snapshots that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mappings</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently removing/flushing the logical
+ rewrite mappings.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing CLOG pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing CLOG pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing CommitTs pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing CommitTs pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing SUBTRANS pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing SUBTRANS pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing MULTIXACT pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing MULTIXACT pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing SLRU pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing SLRU pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing sync requests</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently performing sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old XLOG files</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently recycling old XLOG files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The CHECKPOINTER process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 958220c495..df568eecd8 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -9089,6 +9089,9 @@ CreateCheckPoint(int flags)
if (RecoveryInProgress() && (flags & CHECKPOINT_END_OF_RECOVERY) == 0)
elog(ERROR, "can't create a checkpoint during recovery");
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags);
+
/*
* Prepare to accumulate statistics.
*
@@ -9432,8 +9435,12 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/*
* Make more log segments if needed. (Do this after recycling old log
@@ -9455,6 +9462,9 @@ CreateCheckPoint(int flags)
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ checkpoint_progress_end(flags);
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -9568,29 +9578,60 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
CheckPointPredicate();
+
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
- ProcessSyncRequests();
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FILE_SYNC);
+ ProcessSyncRequests(flags);
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -9727,6 +9768,9 @@ CreateRestartPoint(int flags)
XLogCtl->RedoRecPtr = lastCheckPoint.redo;
SpinLockRelease(&XLogCtl->info_lck);
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags);
+
/*
* Prepare to accumulate statistics.
*
@@ -9837,7 +9881,11 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/*
* Make more log segments if needed. (Do this after recycling old log
@@ -9858,6 +9906,9 @@ CreateRestartPoint(int flags)
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the checkpoint. */
+ checkpoint_progress_end(flags);
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -13242,3 +13293,73 @@ XLogRequestWalReceiverReply(void)
{
doRequestWalReceiverReply = true;
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+void
+checkpoint_progress_start(int flags)
+{
+ /* In bootstrap mode, we don't actually record anything. */
+ if (IsBootstrapProcessingMode())
+ return;
+
+ /*
+ * Cannot access pg_stat_progress_checkpoint view in case of checkpoint
+ * during shutdown and end-of-recovery.
+ */
+ if ((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
+ {
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INIT);
+ if (flags & CHECKPOINT_CAUSE_XLOG)
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
+ PROGRESS_CHECKPOINT_KIND_WAL);
+ else if (flags & CHECKPOINT_CAUSE_TIME)
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
+ PROGRESS_CHECKPOINT_KIND_TIME);
+ else if (flags & CHECKPOINT_FORCE)
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
+ PROGRESS_CHECKPOINT_KIND_FORCE);
+ else
+ checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
+ PROGRESS_CHECKPOINT_KIND_UNKNOWN);
+ }
+}
+
+/*
+ * Update index'th member in st_progress_param[] array with the latest value.
+ */
+void
+checkpoint_progress_update_param(int flags, int index, int64 val)
+{
+ /* In bootstrap mode, we don't actually record anything. */
+ if (IsBootstrapProcessingMode())
+ return;
+
+ /*
+ * Cannot access pg_stat_progress_checkpoint view in case of checkpoint
+ * during shutdown and end-of-recovery.
+ */
+ if ((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
+ pgstat_progress_update_param(index, val);
+}
+
+/*
+ * Stop reporting progress of the checkpoint.
+ */
+void
+checkpoint_progress_end(int flags)
+{
+ /* In bootstrap mode, we don't actually record anything. */
+ if (IsBootstrapProcessingMode())
+ return;
+
+ /*
+ * Cannot access pg_stat_progress_checkpoint view in case of checkpoint
+ * during shutdown and end-of-recovery.
+ */
+ if ((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
+ pgstat_progress_end_command();
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 3cb69b1f87..6a90d63fff 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1286,3 +1286,32 @@ CREATE VIEW pg_stat_subscription_workers AS
FROM pg_subscription_rel) sr,
LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w
JOIN pg_subscription s ON (w.subid = s.oid);
+
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ CASE S.param1 WHEN 0 THEN 'wal'
+ WHEN 1 THEN 'time'
+ WHEN 2 THEN 'force'
+ END AS kind,
+ CASE S.param2 WHEN 0 THEN 'initializing'
+ WHEN 1 THEN 'checkpointing replication slots'
+ WHEN 2 THEN 'checkpointing snapshots'
+ WHEN 3 THEN 'checkpointing logical rewrite mappings'
+ WHEN 4 THEN 'checkpointing CLOG pages'
+ WHEN 5 THEN 'checkpointing CommitTs pages'
+ WHEN 6 THEN 'checkpointing SUBTRANS pages'
+ WHEN 7 THEN 'checkpointing MULTIXACT pages'
+ WHEN 8 THEN 'checkpointing SLRU pages'
+ WHEN 9 THEN 'checkpointing buffers'
+ WHEN 10 THEN 'performing sync requests'
+ WHEN 11 THEN 'performing two phase checkpoint'
+ WHEN 12 THEN 'recycling old XLOG files'
+ WHEN 13 THEN 'Finalizing'
+ END AS phase,
+ S.param3 AS total_buffer_writes,
+ S.param4 AS buffers_processed,
+ S.param5 AS buffers_written,
+ S.param6 AS total_file_syncs,
+ S.param7 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index f5459c68f8..cf0ad299f3 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -38,6 +38,7 @@
#include "access/xlogutils.h"
#include "catalog/catalog.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2012,6 +2013,9 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ checkpoint_progress_update_param(flags,
+ PROGRESS_CHECKPOINT_TOTAL_BUFFER_WRITES,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2129,6 +2133,9 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ checkpoint_progress_update_param(flags,
+ PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2149,6 +2156,9 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.m_buf_written_checkpoints++;
num_written++;
+ checkpoint_progress_update_param(flags,
+ PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index 543f691f2d..b8f6aebb7c 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -277,7 +278,7 @@ SyncPostCheckpoint(void)
* ProcessSyncRequests() -- Process queued fsync requests.
*/
void
-ProcessSyncRequests(void)
+ProcessSyncRequests(int flags)
{
static bool sync_in_progress = false;
@@ -355,6 +356,10 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ checkpoint_progress_update_param(flags,
+ PROGRESS_CHECKPOINT_TOTAL_FILE_SYNCS,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -418,6 +423,9 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ checkpoint_progress_update_param(flags,
+ PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 15cb17ace4..7438e0ce84 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -494,6 +494,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index a4b1c1286f..58c547b2d5 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -353,6 +353,10 @@ extern void do_pg_abort_backup(int code, Datum arg);
extern void register_persistent_abort_backup_handler(void);
extern SessionBackupState get_backup_status(void);
+extern void checkpoint_progress_start(int flags);
+extern void checkpoint_progress_update_param(int flags, int index, int64 val);
+extern void checkpoint_progress_end(int flags);
+
/* File path names (all relative to $PGDATA) */
#define RECOVERY_SIGNAL_FILE "recovery.signal"
#define STANDBY_SIGNAL_FILE "standby.signal"
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..e1c574d053 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,35 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_KIND 0
+#define PROGRESS_CHECKPOINT_PHASE 1
+#define PROGRESS_CHECKPOINT_TOTAL_BUFFER_WRITES 2
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 3
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 4
+#define PROGRESS_CHECKPOINT_TOTAL_FILE_SYNCS 5
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 6
+
+/* Kinds of checkpoint (as advertised via PROGRESS_CHECKPOINT_KIND) */
+#define PROGRESS_CHECKPOINT_KIND_WAL 0
+#define PROGRESS_CHECKPOINT_KIND_TIME 1
+#define PROGRESS_CHECKPOINT_KIND_FORCE 2
+#define PROGRESS_CHECKPOINT_KIND_UNKNOWN 3
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 0
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 1
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 2
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 3
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 4
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 5
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 6
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 9
+#define PROGRESS_CHECKPOINT_PHASE_FILE_SYNC 10
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 11
+#define PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE 12
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 13
+
#endif
diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h
index 9737e1eb67..fed52efa30 100644
--- a/src/include/storage/sync.h
+++ b/src/include/storage/sync.h
@@ -58,7 +58,7 @@ typedef struct FileTag
extern void InitSync(void);
extern void SyncPreCheckpoint(void);
extern void SyncPostCheckpoint(void);
-extern void ProcessSyncRequests(void);
+extern void ProcessSyncRequests(int flags);
extern void RememberSyncRequest(const FileTag *ftag, SyncRequestType type);
extern bool RegisterSyncRequest(const FileTag *ftag, SyncRequestType type,
bool retryOnError);
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
--
2.25.1
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-15 12:15 ` Nitin Jadhav <[email protected]>
2022-02-16 19:51 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-15 12:15 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> Apart from above fields, I am planning to add few more fields to the
> view in the next patch. That is, process ID of the backend process
> which triggered a CHECKPOINT command, checkpoint start location, filed
> to indicate whether it is a checkpoint or restartpoint and elapsed
> time of the checkpoint operation. Please share your thoughts. I would
> be happy to add any other information that contributes to showing the
> progress of checkpoint.
The progress reporting mechanism of postgres uses the
'st_progress_param' array of 'PgBackendStatus' structure to hold the
information related to the progress. There is a function
'pgstat_progress_update_param()' which takes 'index' and 'val' as
arguments and updates the 'val' to corresponding 'index' in the
'st_progress_param' array. This mechanism works fine when all the
progress information is of type integer as the data type of
'st_progress_param' is of type integer. If the progress data is of
different type than integer, then there is no easy way to do so. In my
understanding, define a new structure with additional fields. Add this
as part of the 'PgBackendStatus' structure and support the necessary
function to update and fetch the data from this structure. This
becomes very ugly as it will not match the existing mechanism of
progress reporting. Kindly let me know if there is any better way to
handle this. If there are any changes to the existing mechanism to
make it generic to support basic data types, I would like to discuss
this in the new thread.
On Thu, Feb 10, 2022 at 12:22 PM Nitin Jadhav
<[email protected]> wrote:
>
> > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > >
> > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> >
> > I agree to provide above mentioned information as part of showing the
> > progress of current checkpoint operation. I am currently looking into
> > the code to know if any other information can be added.
>
> Here is the initial patch to show the progress of checkpoint through
> pg_stat_progress_checkpoint view. Please find the attachment.
>
> The information added to this view are pid - process ID of a
> CHECKPOINTER process, kind - kind of checkpoint indicates the reason
> for checkpoint (values can be wal, time or force), phase - indicates
> the current phase of checkpoint operation, total_buffer_writes - total
> number of buffers to be written, buffers_processed - number of buffers
> processed, buffers_written - number of buffers written,
> total_file_syncs - total number of files to be synced, files_synced -
> number of files synced.
>
> There are many operations happen as part of checkpoint. For each of
> the operation I am updating the phase field of
> pg_stat_progress_checkpoint view. The values supported for this field
> are initializing, checkpointing replication slots, checkpointing
> snapshots, checkpointing logical rewrite mappings, checkpointing CLOG
> pages, checkpointing CommitTs pages, checkpointing SUBTRANS pages,
> checkpointing MULTIXACT pages, checkpointing SLRU pages, checkpointing
> buffers, performing sync requests, performing two phase checkpoint,
> recycling old XLOG files and Finalizing. In case of checkpointing
> buffers phase, the fields total_buffer_writes, buffers_processed and
> buffers_written shows the detailed progress of writing buffers. In
> case of performing sync requests phase, the fields total_file_syncs
> and files_synced shows the detailed progress of syncing files. In
> other phases, only the phase field is getting updated and it is
> difficult to show the progress because we do not get the total number
> of files count without traversing the directory. It is not worth to
> calculate that as it affects the performance of the checkpoint. I also
> gave a thought to just mention the number of files processed, but this
> wont give a meaningful progress information (It can be treated as
> statistics). Hence just updating the phase field in those scenarios.
>
> Apart from above fields, I am planning to add few more fields to the
> view in the next patch. That is, process ID of the backend process
> which triggered a CHECKPOINT command, checkpoint start location, filed
> to indicate whether it is a checkpoint or restartpoint and elapsed
> time of the checkpoint operation. Please share your thoughts. I would
> be happy to add any other information that contributes to showing the
> progress of checkpoint.
>
> As per the discussion in this thread, there should be some mechanism
> to show the progress of checkpoint during shutdown and end-of-recovery
> cases as we cannot access pg_stat_progress_checkpoint in those cases.
> I am working on this to use log_startup_progress_interval mechanism to
> log the progress in the server logs.
>
> Kindly review the patch and share your thoughts.
>
>
> On Fri, Jan 28, 2022 at 12:24 PM Bharath Rupireddy
> <[email protected]> wrote:
> >
> > On Fri, Jan 21, 2022 at 11:07 AM Nitin Jadhav
> > <[email protected]> wrote:
> > >
> > > > I think the right choice to solve the *general* problem is the
> > > > mentioned pg_stat_progress_checkpoints.
> > > >
> > > > We may want to *additionally* have the ability to log the progress
> > > > specifically for the special cases when we're not able to use that
> > > > view. And in those case, we can perhaps just use the existing
> > > > log_startup_progress_interval parameter for this as well -- at least
> > > > for the startup checkpoint.
> > >
> > > +1
> > >
> > > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > > >
> > > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> > >
> > > I agree to provide above mentioned information as part of showing the
> > > progress of current checkpoint operation. I am currently looking into
> > > the code to know if any other information can be added.
> >
> > As suggested in the other thread by Julien, I'm changing the subject
> > of this thread to reflect the discussion.
> >
> > Regards,
> > Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-15 12:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-16 19:51 ` Matthias van de Meent <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Matthias van de Meent @ 2022-02-16 19:51 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, 15 Feb 2022 at 13:16, Nitin Jadhav
<[email protected]> wrote:
>
> > Apart from above fields, I am planning to add few more fields to the
> > view in the next patch. That is, process ID of the backend process
> > which triggered a CHECKPOINT command, checkpoint start location, filed
> > to indicate whether it is a checkpoint or restartpoint and elapsed
> > time of the checkpoint operation. Please share your thoughts. I would
> > be happy to add any other information that contributes to showing the
> > progress of checkpoint.
>
> The progress reporting mechanism of postgres uses the
> 'st_progress_param' array of 'PgBackendStatus' structure to hold the
> information related to the progress. There is a function
> 'pgstat_progress_update_param()' which takes 'index' and 'val' as
> arguments and updates the 'val' to corresponding 'index' in the
> 'st_progress_param' array. This mechanism works fine when all the
> progress information is of type integer as the data type of
> 'st_progress_param' is of type integer. If the progress data is of
> different type than integer, then there is no easy way to do so.
Progress parameters are int64, so all of the new 'checkpoint start
location' (lsn = uint64), 'triggering backend PID' (int), 'elapsed
time' (store as start time in stat_progress, timestamp fits in 64
bits) and 'checkpoint or restartpoint?' (boolean) would each fit in a
current stat_progress parameter. Some processing would be required at
the view, but that's not impossible to overcome.
Kind regards,
Matthias van de Meent
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-16 20:02 ` Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 06:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2 siblings, 2 replies; 85+ messages in thread
From: Matthias van de Meent @ 2022-02-16 20:02 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, 10 Feb 2022 at 07:53, Nitin Jadhav
<[email protected]> wrote:
>
> > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > >
> > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> >
> > I agree to provide above mentioned information as part of showing the
> > progress of current checkpoint operation. I am currently looking into
> > the code to know if any other information can be added.
>
> Here is the initial patch to show the progress of checkpoint through
> pg_stat_progress_checkpoint view. Please find the attachment.
>
> The information added to this view are pid - process ID of a
> CHECKPOINTER process, kind - kind of checkpoint indicates the reason
> for checkpoint (values can be wal, time or force), phase - indicates
> the current phase of checkpoint operation, total_buffer_writes - total
> number of buffers to be written, buffers_processed - number of buffers
> processed, buffers_written - number of buffers written,
> total_file_syncs - total number of files to be synced, files_synced -
> number of files synced.
>
> There are many operations happen as part of checkpoint. For each of
> the operation I am updating the phase field of
> pg_stat_progress_checkpoint view. The values supported for this field
> are initializing, checkpointing replication slots, checkpointing
> snapshots, checkpointing logical rewrite mappings, checkpointing CLOG
> pages, checkpointing CommitTs pages, checkpointing SUBTRANS pages,
> checkpointing MULTIXACT pages, checkpointing SLRU pages, checkpointing
> buffers, performing sync requests, performing two phase checkpoint,
> recycling old XLOG files and Finalizing. In case of checkpointing
> buffers phase, the fields total_buffer_writes, buffers_processed and
> buffers_written shows the detailed progress of writing buffers. In
> case of performing sync requests phase, the fields total_file_syncs
> and files_synced shows the detailed progress of syncing files. In
> other phases, only the phase field is getting updated and it is
> difficult to show the progress because we do not get the total number
> of files count without traversing the directory. It is not worth to
> calculate that as it affects the performance of the checkpoint. I also
> gave a thought to just mention the number of files processed, but this
> wont give a meaningful progress information (It can be treated as
> statistics). Hence just updating the phase field in those scenarios.
>
> Apart from above fields, I am planning to add few more fields to the
> view in the next patch. That is, process ID of the backend process
> which triggered a CHECKPOINT command, checkpoint start location, filed
> to indicate whether it is a checkpoint or restartpoint and elapsed
> time of the checkpoint operation. Please share your thoughts. I would
> be happy to add any other information that contributes to showing the
> progress of checkpoint.
>
> As per the discussion in this thread, there should be some mechanism
> to show the progress of checkpoint during shutdown and end-of-recovery
> cases as we cannot access pg_stat_progress_checkpoint in those cases.
> I am working on this to use log_startup_progress_interval mechanism to
> log the progress in the server logs.
>
> Kindly review the patch and share your thoughts.
Interesting idea, and overall a nice addition to the
pg_stat_progress_* reporting infrastructure.
Could you add your patch to the current commitfest at
https://commitfest.postgresql.org/37/?
See below for some comments on the patch:
> xlog.c @ checkpoint_progress_start, checkpoint_progress_update_param, checkpoint_progress_end
> + /* In bootstrap mode, we don't actually record anything. */
> + if (IsBootstrapProcessingMode())
> + return;
Why do you check against the state of the system?
pgstat_progress_update_* already provides protections against updating
the progress tables if the progress infrastructure is not loaded; and
otherwise (in the happy path) the cost of updating the progress fields
will be quite a bit higher than normal. Updating stat_progress isn't
very expensive (quite cheap, really), so I don't quite get why you
guard against reporting stats when you expect no other client to be
listening.
I think you can simplify this a lot by directly using
pgstat_progress_update_param() instead.
> xlog.c @ checkpoint_progress_start
> + pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
> + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_INIT);
> + if (flags & CHECKPOINT_CAUSE_XLOG)
> + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> + PROGRESS_CHECKPOINT_KIND_WAL);
> + else if (flags & CHECKPOINT_CAUSE_TIME)
> + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> + PROGRESS_CHECKPOINT_KIND_TIME);
> + [...]
Could you assign the kind of checkpoint to a local variable, and then
update the "phase" and "kind" parameters at the same time through
pgstat_progress_update_multi_param(2, ...)? See
BuildRelationExtStatistics in extended_stats.c for an example usage.
Note that regardless of whether checkpoint_progress_update* will
remain, the checks done in that function already have been checked in
this function as well, so you can use the pgstat_* functions directly.
> monitoring.sgml
> + <structname>pg_stat_progress_checkpoint</structname> view will contain a
> + single row indicating the progress of checkpoint operation.
... add "if a checkpoint is currently active".
> + <structfield>total_buffer_writes</structfield> <type>bigint</type>
> + <structfield>total_file_syncs</structfield> <type>bigint</type>
The other progress tables use [type]_total as column names for counter
targets (e.g. backup_total for backup_streamed, heap_blks_total for
heap_blks_scanned, etc.). I think that `buffers_total` and
`files_total` would be better column names.
> + The checkpoint operation is requested due to XLOG filling.
+ The checkpoint was started because >max_wal_size< of WAL was written.
> + The checkpoint operation is requested due to timeout.
+ The checkpoint was started due to the expiration of a
>checkpoint_timeout< interval
> + The checkpoint operation is forced even if no XLOG activity has occurred
> + since the last one.
+ Some operation forced a checkpoint.
> + <entry><literal>checkpointing CommitTs pages</literal></entry>
CommitTs -> Commit time stamp
Thanks for working on this.
Kind regards,
Matthias van de Meent
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
@ 2022-02-17 06:56 ` Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 11:11 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
1 sibling, 2 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-17 06:56 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > The progress reporting mechanism of postgres uses the
> > 'st_progress_param' array of 'PgBackendStatus' structure to hold the
> > information related to the progress. There is a function
> > 'pgstat_progress_update_param()' which takes 'index' and 'val' as
> > arguments and updates the 'val' to corresponding 'index' in the
> > 'st_progress_param' array. This mechanism works fine when all the
> > progress information is of type integer as the data type of
> > 'st_progress_param' is of type integer. If the progress data is of
> > different type than integer, then there is no easy way to do so.
>
> Progress parameters are int64, so all of the new 'checkpoint start
> location' (lsn = uint64), 'triggering backend PID' (int), 'elapsed
> time' (store as start time in stat_progress, timestamp fits in 64
> bits) and 'checkpoint or restartpoint?' (boolean) would each fit in a
> current stat_progress parameter. Some processing would be required at
> the view, but that's not impossible to overcome.
Thank you for sharing the information. 'triggering backend PID' (int)
- can be stored without any problem. 'checkpoint or restartpoint?'
(boolean) - can be stored as a integer value like
PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
start time in stat_progress, timestamp fits in 64 bits) - As
Timestamptz is of type int64 internally, so we can store the timestamp
value in the progres parameter and then expose a function like
'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
Timestamptz) as argument and then returns string representing the
elapsed time. This function can be called in the view. Is it
safe/advisable to use int64 type here rather than Timestamptz for this
purpose? 'checkpoint start location' (lsn = uint64) - I feel we
cannot use progress parameters for this case. As assigning uint64 to
int64 type would be an issue for larger values and can lead to hidden
bugs.
Thoughts?
Thanks & Regards,
Nitin Jadhav
On Thu, Feb 17, 2022 at 1:33 AM Matthias van de Meent
<[email protected]> wrote:
>
> On Thu, 10 Feb 2022 at 07:53, Nitin Jadhav
> <[email protected]> wrote:
> >
> > > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > > >
> > > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> > >
> > > I agree to provide above mentioned information as part of showing the
> > > progress of current checkpoint operation. I am currently looking into
> > > the code to know if any other information can be added.
> >
> > Here is the initial patch to show the progress of checkpoint through
> > pg_stat_progress_checkpoint view. Please find the attachment.
> >
> > The information added to this view are pid - process ID of a
> > CHECKPOINTER process, kind - kind of checkpoint indicates the reason
> > for checkpoint (values can be wal, time or force), phase - indicates
> > the current phase of checkpoint operation, total_buffer_writes - total
> > number of buffers to be written, buffers_processed - number of buffers
> > processed, buffers_written - number of buffers written,
> > total_file_syncs - total number of files to be synced, files_synced -
> > number of files synced.
> >
> > There are many operations happen as part of checkpoint. For each of
> > the operation I am updating the phase field of
> > pg_stat_progress_checkpoint view. The values supported for this field
> > are initializing, checkpointing replication slots, checkpointing
> > snapshots, checkpointing logical rewrite mappings, checkpointing CLOG
> > pages, checkpointing CommitTs pages, checkpointing SUBTRANS pages,
> > checkpointing MULTIXACT pages, checkpointing SLRU pages, checkpointing
> > buffers, performing sync requests, performing two phase checkpoint,
> > recycling old XLOG files and Finalizing. In case of checkpointing
> > buffers phase, the fields total_buffer_writes, buffers_processed and
> > buffers_written shows the detailed progress of writing buffers. In
> > case of performing sync requests phase, the fields total_file_syncs
> > and files_synced shows the detailed progress of syncing files. In
> > other phases, only the phase field is getting updated and it is
> > difficult to show the progress because we do not get the total number
> > of files count without traversing the directory. It is not worth to
> > calculate that as it affects the performance of the checkpoint. I also
> > gave a thought to just mention the number of files processed, but this
> > wont give a meaningful progress information (It can be treated as
> > statistics). Hence just updating the phase field in those scenarios.
> >
> > Apart from above fields, I am planning to add few more fields to the
> > view in the next patch. That is, process ID of the backend process
> > which triggered a CHECKPOINT command, checkpoint start location, filed
> > to indicate whether it is a checkpoint or restartpoint and elapsed
> > time of the checkpoint operation. Please share your thoughts. I would
> > be happy to add any other information that contributes to showing the
> > progress of checkpoint.
> >
> > As per the discussion in this thread, there should be some mechanism
> > to show the progress of checkpoint during shutdown and end-of-recovery
> > cases as we cannot access pg_stat_progress_checkpoint in those cases.
> > I am working on this to use log_startup_progress_interval mechanism to
> > log the progress in the server logs.
> >
> > Kindly review the patch and share your thoughts.
>
> Interesting idea, and overall a nice addition to the
> pg_stat_progress_* reporting infrastructure.
>
> Could you add your patch to the current commitfest at
> https://commitfest.postgresql.org/37/?
>
> See below for some comments on the patch:
>
> > xlog.c @ checkpoint_progress_start, checkpoint_progress_update_param, checkpoint_progress_end
> > + /* In bootstrap mode, we don't actually record anything. */
> > + if (IsBootstrapProcessingMode())
> > + return;
>
> Why do you check against the state of the system?
> pgstat_progress_update_* already provides protections against updating
> the progress tables if the progress infrastructure is not loaded; and
> otherwise (in the happy path) the cost of updating the progress fields
> will be quite a bit higher than normal. Updating stat_progress isn't
> very expensive (quite cheap, really), so I don't quite get why you
> guard against reporting stats when you expect no other client to be
> listening.
>
> I think you can simplify this a lot by directly using
> pgstat_progress_update_param() instead.
>
> > xlog.c @ checkpoint_progress_start
> > + pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
> > + PROGRESS_CHECKPOINT_PHASE_INIT);
> > + if (flags & CHECKPOINT_CAUSE_XLOG)
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> > + PROGRESS_CHECKPOINT_KIND_WAL);
> > + else if (flags & CHECKPOINT_CAUSE_TIME)
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> > + PROGRESS_CHECKPOINT_KIND_TIME);
> > + [...]
>
> Could you assign the kind of checkpoint to a local variable, and then
> update the "phase" and "kind" parameters at the same time through
> pgstat_progress_update_multi_param(2, ...)? See
> BuildRelationExtStatistics in extended_stats.c for an example usage.
> Note that regardless of whether checkpoint_progress_update* will
> remain, the checks done in that function already have been checked in
> this function as well, so you can use the pgstat_* functions directly.
>
> > monitoring.sgml
> > + <structname>pg_stat_progress_checkpoint</structname> view will contain a
> > + single row indicating the progress of checkpoint operation.
>
> ... add "if a checkpoint is currently active".
>
> > + <structfield>total_buffer_writes</structfield> <type>bigint</type>
> > + <structfield>total_file_syncs</structfield> <type>bigint</type>
>
> The other progress tables use [type]_total as column names for counter
> targets (e.g. backup_total for backup_streamed, heap_blks_total for
> heap_blks_scanned, etc.). I think that `buffers_total` and
> `files_total` would be better column names.
>
> > + The checkpoint operation is requested due to XLOG filling.
>
> + The checkpoint was started because >max_wal_size< of WAL was written.
>
> > + The checkpoint operation is requested due to timeout.
>
> + The checkpoint was started due to the expiration of a
> >checkpoint_timeout< interval
>
> > + The checkpoint operation is forced even if no XLOG activity has occurred
> > + since the last one.
>
> + Some operation forced a checkpoint.
>
> > + <entry><literal>checkpointing CommitTs pages</literal></entry>
>
> CommitTs -> Commit time stamp
>
> Thanks for working on this.
>
> Kind regards,
>
> Matthias van de Meent
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-17 11:05 ` Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
1 sibling, 1 reply; 85+ messages in thread
From: Julien Rouhaud @ 2022-02-17 11:05 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On Thu, Feb 17, 2022 at 12:26:07PM +0530, Nitin Jadhav wrote:
>
> Thank you for sharing the information. 'triggering backend PID' (int)
> - can be stored without any problem.
There can be multiple processes triggering a checkpoint, or at least wanting it
to happen or happen faster.
> 'checkpoint or restartpoint?'
Do you actually need to store that? Can't it be inferred from
pg_is_in_recovery()?
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-02-17 17:09 ` Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-17 17:09 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > Thank you for sharing the information. 'triggering backend PID' (int)
> > - can be stored without any problem.
>
> There can be multiple processes triggering a checkpoint, or at least wanting it
> to happen or happen faster.
Yes. There can be multiple processes but there will be one checkpoint
operation at a time. So the backend PID corresponds to the current
checkpoint operation. Let me know if I am missing something.
> > 'checkpoint or restartpoint?'
>
> Do you actually need to store that? Can't it be inferred from
> pg_is_in_recovery()?
AFAIK we cannot use pg_is_in_recovery() to predict whether it is a
checkpoint or restartpoint because if the system exits from recovery
mode during restartpoint then any query to pg_stat_progress_checkpoint
view will return it as a checkpoint which is ideally not correct. Please
correct me if I am wrong.
Thanks & Regards,
Nitin Jadhav
On Thu, Feb 17, 2022 at 4:35 PM Julien Rouhaud <[email protected]> wrote:
>
> Hi,
>
> On Thu, Feb 17, 2022 at 12:26:07PM +0530, Nitin Jadhav wrote:
> >
> > Thank you for sharing the information. 'triggering backend PID' (int)
> > - can be stored without any problem.
>
> There can be multiple processes triggering a checkpoint, or at least wanting it
> to happen or happen faster.
>
> > 'checkpoint or restartpoint?'
>
> Do you actually need to store that? Can't it be inferred from
> pg_is_in_recovery()?
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-17 17:27 ` Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Julien Rouhaud @ 2022-02-17 17:27 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On Thu, Feb 17, 2022 at 10:39:02PM +0530, Nitin Jadhav wrote:
> > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > - can be stored without any problem.
> >
> > There can be multiple processes triggering a checkpoint, or at least wanting it
> > to happen or happen faster.
>
> Yes. There can be multiple processes but there will be one checkpoint
> operation at a time. So the backend PID corresponds to the current
> checkpoint operation. Let me know if I am missing something.
If there's a checkpoint timed triggered and then someone calls
pg_start_backup() which then wait for the end of the current checkpoint
(possibly after changing the flags), I think the view should reflect that in
some way. Maybe storing an array of (pid, flags) is too much, but at least a
counter with the number of processes actively waiting for the end of the
checkpoint.
> > > 'checkpoint or restartpoint?'
> >
> > Do you actually need to store that? Can't it be inferred from
> > pg_is_in_recovery()?
>
> AFAIK we cannot use pg_is_in_recovery() to predict whether it is a
> checkpoint or restartpoint because if the system exits from recovery
> mode during restartpoint then any query to pg_stat_progress_checkpoint
> view will return it as a checkpoint which is ideally not correct. Please
> correct me if I am wrong.
Recovery ends with an end-of-recovery checkpoint that has to finish before the
promotion can happen, so I don't think that a restart can still be in progress
if pg_is_in_recovery() returns false.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-02-18 06:50 ` Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-18 06:50 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > > - can be stored without any problem.
> > >
> > > There can be multiple processes triggering a checkpoint, or at least wanting it
> > > to happen or happen faster.
> >
> > Yes. There can be multiple processes but there will be one checkpoint
> > operation at a time. So the backend PID corresponds to the current
> > checkpoint operation. Let me know if I am missing something.
>
> If there's a checkpoint timed triggered and then someone calls
> pg_start_backup() which then wait for the end of the current checkpoint
> (possibly after changing the flags), I think the view should reflect that in
> some way. Maybe storing an array of (pid, flags) is too much, but at least a
> counter with the number of processes actively waiting for the end of the
> checkpoint.
Okay. I feel this can be added as additional field but it will not
replace backend_pid field as this represents the pid of the backend
which triggered the current checkpoint. Probably a new field named
'processes_wiating' or 'events_waiting' can be added for this purpose.
Thoughts?
> > > > 'checkpoint or restartpoint?'
> > >
> > > Do you actually need to store that? Can't it be inferred from
> > > pg_is_in_recovery()?
> >
> > AFAIK we cannot use pg_is_in_recovery() to predict whether it is a
> > checkpoint or restartpoint because if the system exits from recovery
> > mode during restartpoint then any query to pg_stat_progress_checkpoint
> > view will return it as a checkpoint which is ideally not correct. Please
> > correct me if I am wrong.
>
> Recovery ends with an end-of-recovery checkpoint that has to finish before the
> promotion can happen, so I don't think that a restart can still be in progress
> if pg_is_in_recovery() returns false.
Probably writing of buffers or syncing files may complete before
pg_is_in_recovery() returns false. But there are some cleanup
operations happen as part of the checkpoint. During this scenario, we
may get false value for pg_is_in_recovery(). Please refer following
piece of code which is present in CreateRestartpoint().
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
Thanks & Regards,
Nitin Jadhav
On Thu, Feb 17, 2022 at 10:57 PM Julien Rouhaud <[email protected]> wrote:
>
> Hi,
>
> On Thu, Feb 17, 2022 at 10:39:02PM +0530, Nitin Jadhav wrote:
> > > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > > - can be stored without any problem.
> > >
> > > There can be multiple processes triggering a checkpoint, or at least wanting it
> > > to happen or happen faster.
> >
> > Yes. There can be multiple processes but there will be one checkpoint
> > operation at a time. So the backend PID corresponds to the current
> > checkpoint operation. Let me know if I am missing something.
>
> If there's a checkpoint timed triggered and then someone calls
> pg_start_backup() which then wait for the end of the current checkpoint
> (possibly after changing the flags), I think the view should reflect that in
> some way. Maybe storing an array of (pid, flags) is too much, but at least a
> counter with the number of processes actively waiting for the end of the
> checkpoint.
>
> > > > 'checkpoint or restartpoint?'
> > >
> > > Do you actually need to store that? Can't it be inferred from
> > > pg_is_in_recovery()?
> >
> > AFAIK we cannot use pg_is_in_recovery() to predict whether it is a
> > checkpoint or restartpoint because if the system exits from recovery
> > mode during restartpoint then any query to pg_stat_progress_checkpoint
> > view will return it as a checkpoint which is ideally not correct. Please
> > correct me if I am wrong.
>
> Recovery ends with an end-of-recovery checkpoint that has to finish before the
> promotion can happen, so I don't think that a restart can still be in progress
> if pg_is_in_recovery() returns false.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-18 07:43 ` Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Julien Rouhaud @ 2022-02-18 07:43 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On Fri, Feb 18, 2022 at 12:20:26PM +0530, Nitin Jadhav wrote:
> >
> > If there's a checkpoint timed triggered and then someone calls
> > pg_start_backup() which then wait for the end of the current checkpoint
> > (possibly after changing the flags), I think the view should reflect that in
> > some way. Maybe storing an array of (pid, flags) is too much, but at least a
> > counter with the number of processes actively waiting for the end of the
> > checkpoint.
>
> Okay. I feel this can be added as additional field but it will not
> replace backend_pid field as this represents the pid of the backend
> which triggered the current checkpoint.
I don't think that's true. Requesting a checkpoint means telling the
checkpointer that it should wake up and start a checkpoint (or restore point)
if it's not already doing so, so the pid will always be the checkpointer pid.
The only exception is a standalone backend, but in that case you won't be able
to query that view anyway.
And also while looking at the patch I see there's the same problem that I
mentioned in the previous thread, which is that the effective flags can be
updated once the checkpoint started, and as-is the view won't reflect that. It
also means that you can't simply display one of wal, time or force but a
possible combination of the flags (including the one not handled in v1).
> Probably a new field named 'processes_wiating' or 'events_waiting' can be
> added for this purpose.
Maybe num_process_waiting?
> > > > > 'checkpoint or restartpoint?'
> > > >
> > > > Do you actually need to store that? Can't it be inferred from
> > > > pg_is_in_recovery()?
> > >
> > > AFAIK we cannot use pg_is_in_recovery() to predict whether it is a
> > > checkpoint or restartpoint because if the system exits from recovery
> > > mode during restartpoint then any query to pg_stat_progress_checkpoint
> > > view will return it as a checkpoint which is ideally not correct. Please
> > > correct me if I am wrong.
> >
> > Recovery ends with an end-of-recovery checkpoint that has to finish before the
> > promotion can happen, so I don't think that a restart can still be in progress
> > if pg_is_in_recovery() returns false.
>
> Probably writing of buffers or syncing files may complete before
> pg_is_in_recovery() returns false. But there are some cleanup
> operations happen as part of the checkpoint. During this scenario, we
> may get false value for pg_is_in_recovery(). Please refer following
> piece of code which is present in CreateRestartpoint().
>
> if (!RecoveryInProgress())
> replayTLI = XLogCtl->InsertTimeLineID;
Then maybe we could store the timeline rather then then kind of checkpoint?
You should still be able to compute the information while giving a bit more
information for the same memory usage.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-02-18 14:37 ` Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-18 14:37 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > Okay. I feel this can be added as additional field but it will not
> > replace backend_pid field as this represents the pid of the backend
> > which triggered the current checkpoint.
>
> I don't think that's true. Requesting a checkpoint means telling the
> checkpointer that it should wake up and start a checkpoint (or restore point)
> if it's not already doing so, so the pid will always be the checkpointer pid.
> The only exception is a standalone backend, but in that case you won't be able
> to query that view anyway.
Yes. I agree that the checkpoint will always be performed by the
checkpointer process. So the pid in the pg_stat_progress_checkpoint
view will always correspond to the checkpointer pid only. Checkpoints
get triggered in many scenarios. One of the cases is the CHECKPOINT
command issued explicitly by the backend. In this scenario I would
like to know the backend pid which triggered the checkpoint. Hence I
would like to add a backend_pid field. So the
pg_stat_progress_checkpoint view contains pid fields as well as
backend_pid fields. The backend_pid contains a valid value only during
the CHECKPOINT command issued by the backend explicitly, otherwise the
value will be 0. We may have to add an additional field to
'CheckpointerShmemStruct' to hold the backend pid. The backend
requesting the checkpoint will update its pid to this structure.
Kindly let me know if you still feel the backend_pid field is not
necessary.
> And also while looking at the patch I see there's the same problem that I
> mentioned in the previous thread, which is that the effective flags can be
> updated once the checkpoint started, and as-is the view won't reflect that. It
> also means that you can't simply display one of wal, time or force but a
> possible combination of the flags (including the one not handled in v1).
If I understand the above comment properly, it has 2 points. First is
to display the combination of flags rather than just displaying wal,
time or force - The idea behind this is to just let the user know the
reason for checkpointing. That is, the checkpoint is started because
max_wal_size is reached or checkpoint_timeout expired or explicitly
issued CHECKPOINT command. The other flags like CHECKPOINT_IMMEDIATE,
CHECKPOINT_WAIT or CHECKPOINT_FLUSH_ALL indicate how the checkpoint
has to be performed. Hence I have not included those in the view. If
it is really required, I would like to modify the code to include
other flags and display the combination. Second point is to reflect
the updated flags in the view. AFAIK, there is a possibility that the
flags get updated during the on-going checkpoint but the reason for
checkpoint (wal, time or force) will remain same for the current
checkpoint. There might be a change in how checkpoint has to be
performed if CHECKPOINT_IMMEDIATE flag is set. If we go with
displaying the combination of flags in the view, then probably we may
have to reflect this in the view.
> > Probably a new field named 'processes_wiating' or 'events_waiting' can be
> > added for this purpose.
>
> Maybe num_process_waiting?
I feel 'processes_wiating' aligns more with the naming conventions of
the fields of the existing progres views.
> > Probably writing of buffers or syncing files may complete before
> > pg_is_in_recovery() returns false. But there are some cleanup
> > operations happen as part of the checkpoint. During this scenario, we
> > may get false value for pg_is_in_recovery(). Please refer following
> > piece of code which is present in CreateRestartpoint().
> >
> > if (!RecoveryInProgress())
> > replayTLI = XLogCtl->InsertTimeLineID;
>
> Then maybe we could store the timeline rather then then kind of checkpoint?
> You should still be able to compute the information while giving a bit more
> information for the same memory usage.
Can you please describe more about how checkpoint/restartpoint can be
confirmed using the timeline id.
Thanks & Regards,
Nitin Jadhav
On Fri, Feb 18, 2022 at 1:13 PM Julien Rouhaud <[email protected]> wrote:
>
> Hi,
>
> On Fri, Feb 18, 2022 at 12:20:26PM +0530, Nitin Jadhav wrote:
> > >
> > > If there's a checkpoint timed triggered and then someone calls
> > > pg_start_backup() which then wait for the end of the current checkpoint
> > > (possibly after changing the flags), I think the view should reflect that in
> > > some way. Maybe storing an array of (pid, flags) is too much, but at least a
> > > counter with the number of processes actively waiting for the end of the
> > > checkpoint.
> >
> > Okay. I feel this can be added as additional field but it will not
> > replace backend_pid field as this represents the pid of the backend
> > which triggered the current checkpoint.
>
> I don't think that's true. Requesting a checkpoint means telling the
> checkpointer that it should wake up and start a checkpoint (or restore point)
> if it's not already doing so, so the pid will always be the checkpointer pid.
> The only exception is a standalone backend, but in that case you won't be able
> to query that view anyway.
>
> And also while looking at the patch I see there's the same problem that I
> mentioned in the previous thread, which is that the effective flags can be
> updated once the checkpoint started, and as-is the view won't reflect that. It
> also means that you can't simply display one of wal, time or force but a
> possible combination of the flags (including the one not handled in v1).
>
> > Probably a new field named 'processes_wiating' or 'events_waiting' can be
> > added for this purpose.
>
> Maybe num_process_waiting?
>
> > > > > > 'checkpoint or restartpoint?'
> > > > >
> > > > > Do you actually need to store that? Can't it be inferred from
> > > > > pg_is_in_recovery()?
> > > >
> > > > AFAIK we cannot use pg_is_in_recovery() to predict whether it is a
> > > > checkpoint or restartpoint because if the system exits from recovery
> > > > mode during restartpoint then any query to pg_stat_progress_checkpoint
> > > > view will return it as a checkpoint which is ideally not correct. Please
> > > > correct me if I am wrong.
> > >
> > > Recovery ends with an end-of-recovery checkpoint that has to finish before the
> > > promotion can happen, so I don't think that a restart can still be in progress
> > > if pg_is_in_recovery() returns false.
> >
> > Probably writing of buffers or syncing files may complete before
> > pg_is_in_recovery() returns false. But there are some cleanup
> > operations happen as part of the checkpoint. During this scenario, we
> > may get false value for pg_is_in_recovery(). Please refer following
> > piece of code which is present in CreateRestartpoint().
> >
> > if (!RecoveryInProgress())
> > replayTLI = XLogCtl->InsertTimeLineID;
>
> Then maybe we could store the timeline rather then then kind of checkpoint?
> You should still be able to compute the information while giving a bit more
> information for the same memory usage.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-19 05:32 ` Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Julien Rouhaud @ 2022-02-19 05:32 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On Fri, Feb 18, 2022 at 08:07:05PM +0530, Nitin Jadhav wrote:
>
> The backend_pid contains a valid value only during
> the CHECKPOINT command issued by the backend explicitly, otherwise the
> value will be 0. We may have to add an additional field to
> 'CheckpointerShmemStruct' to hold the backend pid. The backend
> requesting the checkpoint will update its pid to this structure.
> Kindly let me know if you still feel the backend_pid field is not
> necessary.
There are more scenarios where you can have a baackend requesting a checkpoint
and waiting for its completion, and there may be more than one backend
concerned, so I don't think that storing only one / the first backend pid is
ok.
> > And also while looking at the patch I see there's the same problem that I
> > mentioned in the previous thread, which is that the effective flags can be
> > updated once the checkpoint started, and as-is the view won't reflect that. It
> > also means that you can't simply display one of wal, time or force but a
> > possible combination of the flags (including the one not handled in v1).
>
> If I understand the above comment properly, it has 2 points. First is
> to display the combination of flags rather than just displaying wal,
> time or force - The idea behind this is to just let the user know the
> reason for checkpointing. That is, the checkpoint is started because
> max_wal_size is reached or checkpoint_timeout expired or explicitly
> issued CHECKPOINT command. The other flags like CHECKPOINT_IMMEDIATE,
> CHECKPOINT_WAIT or CHECKPOINT_FLUSH_ALL indicate how the checkpoint
> has to be performed. Hence I have not included those in the view. If
> it is really required, I would like to modify the code to include
> other flags and display the combination.
I think all the information should be exposed. Only knowing why the current
checkpoint has been triggered without any further information seems a bit
useless. Think for instance for cases like [1].
> Second point is to reflect
> the updated flags in the view. AFAIK, there is a possibility that the
> flags get updated during the on-going checkpoint but the reason for
> checkpoint (wal, time or force) will remain same for the current
> checkpoint. There might be a change in how checkpoint has to be
> performed if CHECKPOINT_IMMEDIATE flag is set. If we go with
> displaying the combination of flags in the view, then probably we may
> have to reflect this in the view.
You can only "upgrade" a checkpoint, but not "downgrade" it. So if for
instance you find both CHECKPOINT_CAUSE_TIME and CHECKPOINT_FORCE (which is
possible) you can easily know which one was the one that triggered the
checkpoint and which one was added later.
> > > Probably a new field named 'processes_wiating' or 'events_waiting' can be
> > > added for this purpose.
> >
> > Maybe num_process_waiting?
>
> I feel 'processes_wiating' aligns more with the naming conventions of
> the fields of the existing progres views.
There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
a strong opinion on it, just make sure to correct the typo.
> > > Probably writing of buffers or syncing files may complete before
> > > pg_is_in_recovery() returns false. But there are some cleanup
> > > operations happen as part of the checkpoint. During this scenario, we
> > > may get false value for pg_is_in_recovery(). Please refer following
> > > piece of code which is present in CreateRestartpoint().
> > >
> > > if (!RecoveryInProgress())
> > > replayTLI = XLogCtl->InsertTimeLineID;
> >
> > Then maybe we could store the timeline rather then then kind of checkpoint?
> > You should still be able to compute the information while giving a bit more
> > information for the same memory usage.
>
> Can you please describe more about how checkpoint/restartpoint can be
> confirmed using the timeline id.
If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
restartpoint if the checkpoint's timeline is different from the current
timeline?
[1] https://www.postgresql.org/message-id/1486805889.24568.96.camel%40credativ.de
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-02-22 06:38 ` Nitin Jadhav <[email protected]>
2022-02-22 18:42 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 2 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-22 06:38 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > Thank you for sharing the information. 'triggering backend PID' (int)
> > - can be stored without any problem. 'checkpoint or restartpoint?'
> > (boolean) - can be stored as a integer value like
> > PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
> > PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
> > start time in stat_progress, timestamp fits in 64 bits) - As
> > Timestamptz is of type int64 internally, so we can store the timestamp
> > value in the progres parameter and then expose a function like
> > 'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
> > Timestamptz) as argument and then returns string representing the
> > elapsed time.
>
> No need to use a string there; I think exposing the checkpoint start
> time is good enough. The conversion of int64 to timestamp[tz] can be
> done in SQL (although I'm not sure that exposing the internal bitwise
> representation of Interval should be exposed to that extent) [0].
> Users can then extract the duration interval using now() - start_time,
> which also allows the user to use their own preferred formatting.
The reason for showing the elapsed time rather than exposing the
timestamp directly is in case of checkpoint during shutdown and
end-of-recovery, I am planning to log a message in server logs using
'log_startup_progress_interval' infrastructure which displays elapsed
time. So just to match both of the behaviour I am displaying elapsed
time here. I feel that elapsed time gives a quicker feel of the
progress. Kindly let me know if you still feel just exposing the
timestamp is better than showing the elapsed time.
> > 'checkpoint start location' (lsn = uint64) - I feel we
> > cannot use progress parameters for this case. As assigning uint64 to
> > int64 type would be an issue for larger values and can lead to hidden
> > bugs.
>
> Not necessarily - we can (without much trouble) do a bitwise cast from
> uint64 to int64, and then (in SQL) cast it back to a pg_lsn [1]. Not
> very elegant, but it works quite well.
>
> [1] SELECT '0/0'::pg_lsn + ((CASE WHEN stat.my_int64 < 0 THEN
> pow(2::numeric, 64::numeric)::numeric ELSE 0::numeric END) +
> stat.my_int64::numeric) FROM (SELECT -2::bigint /* 0xFFFFFFFF/FFFFFFFE
> */ AS my_bigint_lsn) AS stat(my_int64);
Thanks for sharing. It works. I will include this in the next patch.
On Sat, Feb 19, 2022 at 11:02 AM Julien Rouhaud <[email protected]> wrote:
>
> Hi,
>
> On Fri, Feb 18, 2022 at 08:07:05PM +0530, Nitin Jadhav wrote:
> >
> > The backend_pid contains a valid value only during
> > the CHECKPOINT command issued by the backend explicitly, otherwise the
> > value will be 0. We may have to add an additional field to
> > 'CheckpointerShmemStruct' to hold the backend pid. The backend
> > requesting the checkpoint will update its pid to this structure.
> > Kindly let me know if you still feel the backend_pid field is not
> > necessary.
>
> There are more scenarios where you can have a baackend requesting a checkpoint
> and waiting for its completion, and there may be more than one backend
> concerned, so I don't think that storing only one / the first backend pid is
> ok.
>
> > > And also while looking at the patch I see there's the same problem that I
> > > mentioned in the previous thread, which is that the effective flags can be
> > > updated once the checkpoint started, and as-is the view won't reflect that. It
> > > also means that you can't simply display one of wal, time or force but a
> > > possible combination of the flags (including the one not handled in v1).
> >
> > If I understand the above comment properly, it has 2 points. First is
> > to display the combination of flags rather than just displaying wal,
> > time or force - The idea behind this is to just let the user know the
> > reason for checkpointing. That is, the checkpoint is started because
> > max_wal_size is reached or checkpoint_timeout expired or explicitly
> > issued CHECKPOINT command. The other flags like CHECKPOINT_IMMEDIATE,
> > CHECKPOINT_WAIT or CHECKPOINT_FLUSH_ALL indicate how the checkpoint
> > has to be performed. Hence I have not included those in the view. If
> > it is really required, I would like to modify the code to include
> > other flags and display the combination.
>
> I think all the information should be exposed. Only knowing why the current
> checkpoint has been triggered without any further information seems a bit
> useless. Think for instance for cases like [1].
>
> > Second point is to reflect
> > the updated flags in the view. AFAIK, there is a possibility that the
> > flags get updated during the on-going checkpoint but the reason for
> > checkpoint (wal, time or force) will remain same for the current
> > checkpoint. There might be a change in how checkpoint has to be
> > performed if CHECKPOINT_IMMEDIATE flag is set. If we go with
> > displaying the combination of flags in the view, then probably we may
> > have to reflect this in the view.
>
> You can only "upgrade" a checkpoint, but not "downgrade" it. So if for
> instance you find both CHECKPOINT_CAUSE_TIME and CHECKPOINT_FORCE (which is
> possible) you can easily know which one was the one that triggered the
> checkpoint and which one was added later.
>
> > > > Probably a new field named 'processes_wiating' or 'events_waiting' can be
> > > > added for this purpose.
> > >
> > > Maybe num_process_waiting?
> >
> > I feel 'processes_wiating' aligns more with the naming conventions of
> > the fields of the existing progres views.
>
> There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
> a strong opinion on it, just make sure to correct the typo.
>
> > > > Probably writing of buffers or syncing files may complete before
> > > > pg_is_in_recovery() returns false. But there are some cleanup
> > > > operations happen as part of the checkpoint. During this scenario, we
> > > > may get false value for pg_is_in_recovery(). Please refer following
> > > > piece of code which is present in CreateRestartpoint().
> > > >
> > > > if (!RecoveryInProgress())
> > > > replayTLI = XLogCtl->InsertTimeLineID;
> > >
> > > Then maybe we could store the timeline rather then then kind of checkpoint?
> > > You should still be able to compute the information while giving a bit more
> > > information for the same memory usage.
> >
> > Can you please describe more about how checkpoint/restartpoint can be
> > confirmed using the timeline id.
>
> If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
> restartpoint if the checkpoint's timeline is different from the current
> timeline?
>
> [1] https://www.postgresql.org/message-id/1486805889.24568.96.camel%40credativ.de
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-22 18:42 ` Matthias van de Meent <[email protected]>
2022-02-23 14:23 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
1 sibling, 1 reply; 85+ messages in thread
From: Matthias van de Meent @ 2022-02-22 18:42 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Tue, 22 Feb 2022 at 07:39, Nitin Jadhav
<[email protected]> wrote:
>
> > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > - can be stored without any problem. 'checkpoint or restartpoint?'
> > > (boolean) - can be stored as a integer value like
> > > PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
> > > PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
> > > start time in stat_progress, timestamp fits in 64 bits) - As
> > > Timestamptz is of type int64 internally, so we can store the timestamp
> > > value in the progres parameter and then expose a function like
> > > 'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
> > > Timestamptz) as argument and then returns string representing the
> > > elapsed time.
> >
> > No need to use a string there; I think exposing the checkpoint start
> > time is good enough. The conversion of int64 to timestamp[tz] can be
> > done in SQL (although I'm not sure that exposing the internal bitwise
> > representation of Interval should be exposed to that extent) [0].
> > Users can then extract the duration interval using now() - start_time,
> > which also allows the user to use their own preferred formatting.
>
> The reason for showing the elapsed time rather than exposing the
> timestamp directly is in case of checkpoint during shutdown and
> end-of-recovery, I am planning to log a message in server logs using
> 'log_startup_progress_interval' infrastructure which displays elapsed
> time. So just to match both of the behaviour I am displaying elapsed
> time here. I feel that elapsed time gives a quicker feel of the
> progress. Kindly let me know if you still feel just exposing the
> timestamp is better than showing the elapsed time.
At least for pg_stat_progress_checkpoint, storing only a timestamp in
the pg_stat storage (instead of repeatedly updating the field as a
duration) seems to provide much more precise measures of 'time
elapsed' for other sessions if one step of the checkpoint is taking a
long time.
I understand the want to integrate the log-based reporting in the same
API, but I don't think that is necessarily the right approach:
pg_stat_progress_* has low-overhead infrastructure specifically to
ensure that most tasks will not run much slower while reporting, never
waiting for locks. Logging, however, needs to take locks (if only to
prevent concurrent writes to the output file at a kernel level) and
thus has a not insignificant overhead and thus is not very useful for
precise and very frequent statistics updates.
So, although similar in nature, I don't think it is smart to use the
exact same infrastructure between pgstat_progress*-based reporting and
log-based progress reporting, especially if your logging-based
progress reporting is not intended to be a debugging-only
configuration option similar to log_min_messages=DEBUG[1..5].
- Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-22 18:42 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
@ 2022-02-23 14:23 ` Nitin Jadhav <[email protected]>
2022-02-23 19:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-23 14:23 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> At least for pg_stat_progress_checkpoint, storing only a timestamp in
> the pg_stat storage (instead of repeatedly updating the field as a
> duration) seems to provide much more precise measures of 'time
> elapsed' for other sessions if one step of the checkpoint is taking a
> long time.
I am storing the checkpoint start timestamp in the st_progress_param[]
and this gets set only once during the checkpoint (at the start of the
checkpoint). I have added function
pg_stat_get_progress_checkpoint_elapsed() which calculates the elapsed
time and returns a string. This function gets called whenever
pg_stat_progress_checkpoint view is queried. Kindly refer v2 patch and
share your thoughts.
> I understand the want to integrate the log-based reporting in the same
> API, but I don't think that is necessarily the right approach:
> pg_stat_progress_* has low-overhead infrastructure specifically to
> ensure that most tasks will not run much slower while reporting, never
> waiting for locks. Logging, however, needs to take locks (if only to
> prevent concurrent writes to the output file at a kernel level) and
> thus has a not insignificant overhead and thus is not very useful for
> precise and very frequent statistics updates.
I understand that the log based reporting is very costly and very
frequent updates are not advisable. I am planning to use the existing
infrastructure of 'log_startup_progress_interval' which provides an
option for the user to configure the interval between each progress
update. Hence it avoids frequent updates to server logs. This approach
is used only during shutdown and end-of-recovery cases because we
cannot access pg_stat_progress_checkpoint view during those scenarios.
> So, although similar in nature, I don't think it is smart to use the
> exact same infrastructure between pgstat_progress*-based reporting and
> log-based progress reporting, especially if your logging-based
> progress reporting is not intended to be a debugging-only
> configuration option similar to log_min_messages=DEBUG[1..5].
Yes. I agree that we cannot use the same infrastructure for both.
Progress views and servers logs have different APIs to report the
progress information. But since both of this are required for the same
purpose, I am planning to use a common function which increases the
code readability than calling it separately in all the scenarios. I am
planning to include log based reporting in the next patch. Even after
that if using the same function is not recommended, I am happy to
change.
Thanks & Regards,
Nitin Jadhav
On Wed, Feb 23, 2022 at 12:13 AM Matthias van de Meent
<[email protected]> wrote:
>
> On Tue, 22 Feb 2022 at 07:39, Nitin Jadhav
> <[email protected]> wrote:
> >
> > > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > > - can be stored without any problem. 'checkpoint or restartpoint?'
> > > > (boolean) - can be stored as a integer value like
> > > > PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
> > > > PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
> > > > start time in stat_progress, timestamp fits in 64 bits) - As
> > > > Timestamptz is of type int64 internally, so we can store the timestamp
> > > > value in the progres parameter and then expose a function like
> > > > 'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
> > > > Timestamptz) as argument and then returns string representing the
> > > > elapsed time.
> > >
> > > No need to use a string there; I think exposing the checkpoint start
> > > time is good enough. The conversion of int64 to timestamp[tz] can be
> > > done in SQL (although I'm not sure that exposing the internal bitwise
> > > representation of Interval should be exposed to that extent) [0].
> > > Users can then extract the duration interval using now() - start_time,
> > > which also allows the user to use their own preferred formatting.
> >
> > The reason for showing the elapsed time rather than exposing the
> > timestamp directly is in case of checkpoint during shutdown and
> > end-of-recovery, I am planning to log a message in server logs using
> > 'log_startup_progress_interval' infrastructure which displays elapsed
> > time. So just to match both of the behaviour I am displaying elapsed
> > time here. I feel that elapsed time gives a quicker feel of the
> > progress. Kindly let me know if you still feel just exposing the
> > timestamp is better than showing the elapsed time.
>
> At least for pg_stat_progress_checkpoint, storing only a timestamp in
> the pg_stat storage (instead of repeatedly updating the field as a
> duration) seems to provide much more precise measures of 'time
> elapsed' for other sessions if one step of the checkpoint is taking a
> long time.
>
> I understand the want to integrate the log-based reporting in the same
> API, but I don't think that is necessarily the right approach:
> pg_stat_progress_* has low-overhead infrastructure specifically to
> ensure that most tasks will not run much slower while reporting, never
> waiting for locks. Logging, however, needs to take locks (if only to
> prevent concurrent writes to the output file at a kernel level) and
> thus has a not insignificant overhead and thus is not very useful for
> precise and very frequent statistics updates.
>
> So, although similar in nature, I don't think it is smart to use the
> exact same infrastructure between pgstat_progress*-based reporting and
> log-based progress reporting, especially if your logging-based
> progress reporting is not intended to be a debugging-only
> configuration option similar to log_min_messages=DEBUG[1..5].
>
> - Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-22 18:42 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-23 14:23 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-23 19:50 ` Matthias van de Meent <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Matthias van de Meent @ 2022-02-23 19:50 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>
On Wed, 23 Feb 2022 at 15:24, Nitin Jadhav
<[email protected]> wrote:
>
> > At least for pg_stat_progress_checkpoint, storing only a timestamp in
> > the pg_stat storage (instead of repeatedly updating the field as a
> > duration) seems to provide much more precise measures of 'time
> > elapsed' for other sessions if one step of the checkpoint is taking a
> > long time.
>
> I am storing the checkpoint start timestamp in the st_progress_param[]
> and this gets set only once during the checkpoint (at the start of the
> checkpoint). I have added function
> pg_stat_get_progress_checkpoint_elapsed() which calculates the elapsed
> time and returns a string. This function gets called whenever
> pg_stat_progress_checkpoint view is queried. Kindly refer v2 patch and
> share your thoughts.
I dislike the lack of access to the actual value of the checkpoint
start / checkpoint elapsed field.
As a user, if I query the pg_stat_progress_* views, my terminal or
application can easily interpret an `interval` value and cast it to
string, but the opposite is not true: the current implementation for
pg_stat_get_progress_checkpoint_elapsed loses precision. This is why
we use typed numeric fields in effectively all other places instead of
stringified versions of the values: oid fields, counters, etc are all
rendered as bigint in the view, so that no information is lost and
interpretation is trivial.
> > I understand the want to integrate the log-based reporting in the same
> > API, but I don't think that is necessarily the right approach:
> > pg_stat_progress_* has low-overhead infrastructure specifically to
> > ensure that most tasks will not run much slower while reporting, never
> > waiting for locks. Logging, however, needs to take locks (if only to
> > prevent concurrent writes to the output file at a kernel level) and
> > thus has a not insignificant overhead and thus is not very useful for
> > precise and very frequent statistics updates.
>
> I understand that the log based reporting is very costly and very
> frequent updates are not advisable. I am planning to use the existing
> infrastructure of 'log_startup_progress_interval' which provides an
> option for the user to configure the interval between each progress
> update. Hence it avoids frequent updates to server logs. This approach
> is used only during shutdown and end-of-recovery cases because we
> cannot access pg_stat_progress_checkpoint view during those scenarios.
I see; but log_startup_progress_interval seems to be exclusively
consumed through the ereport_startup_progress macro. Why put
startup/shutdown logging on the same path as the happy flow of normal
checkpoints?
> > So, although similar in nature, I don't think it is smart to use the
> > exact same infrastructure between pgstat_progress*-based reporting and
> > log-based progress reporting, especially if your logging-based
> > progress reporting is not intended to be a debugging-only
> > configuration option similar to log_min_messages=DEBUG[1..5].
>
> Yes. I agree that we cannot use the same infrastructure for both.
> Progress views and servers logs have different APIs to report the
> progress information. But since both of this are required for the same
> purpose, I am planning to use a common function which increases the
> code readability than calling it separately in all the scenarios. I am
> planning to include log based reporting in the next patch. Even after
> that if using the same function is not recommended, I am happy to
> change.
I don't think that checkpoint_progress_update_param(int, uint64) fits
well with the construction of progress log messages, requiring
special-casing / matching the offset numbers to actual fields inside
that single function, which adds unnecessary overhead when compared
against normal and direct calls to the related infrastructure.
I think that, instead of looking to what might at some point be added,
it is better to use the currently available functions instead, and
move to new functions if and when the log-based reporting requires it.
- Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-23 13:28 ` Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-23 18:39 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Alvaro Herrera <[email protected]>
2022-02-23 19:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Justin Pryzby <[email protected]>
2022-02-24 17:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
1 sibling, 4 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-23 13:28 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> I will make use of pgstat_progress_update_multi_param() in the next
> patch to replace multiple calls to checkpoint_progress_update_param().
Fixed.
---
> > The other progress tables use [type]_total as column names for counter
> > targets (e.g. backup_total for backup_streamed, heap_blks_total for
> > heap_blks_scanned, etc.). I think that `buffers_total` and
> > `files_total` would be better column names.
>
> I agree and I will update this in the next patch.
Fixed.
---
> How about this "The checkpoint is started because max_wal_size is reached".
>
> "The checkpoint is started because checkpoint_timeout expired".
>
> "The checkpoint is started because some operation forced a checkpoint".
I have used the above description. Kindly let me know if any changes
are required.
---
> > > + <entry><literal>checkpointing CommitTs pages</literal></entry>
> >
> > CommitTs -> Commit time stamp
>
> I will handle this in the next patch.
Fixed.
---
> There are more scenarios where you can have a baackend requesting a checkpoint
> and waiting for its completion, and there may be more than one backend
> concerned, so I don't think that storing only one / the first backend pid is
> ok.
Thanks for this information. I am not considering backend_pid.
---
> I think all the information should be exposed. Only knowing why the current
> checkpoint has been triggered without any further information seems a bit
> useless. Think for instance for cases like [1].
I have supported all possible checkpoint kinds. Added
pg_stat_get_progress_checkpoint_kind() to convert the flags (int) to a
string representing a combination of flags and also checking for the
flag update in ImmediateCheckpointRequested() which checks whether
CHECKPOINT_IMMEDIATE flag is set or not. I did not find any other
cases where the flags get changed (which changes the current
checkpoint behaviour) during the checkpoint. Kindly let me know if I
am missing something.
---
> > I feel 'processes_wiating' aligns more with the naming conventions of
> > the fields of the existing progres views.
>
> There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
> a strong opinion on it, just make sure to correct the typo.
More analysis is required to support this. I am planning to take care
in the next patch.
---
> If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
> restartpoint if the checkpoint's timeline is different from the current
> timeline?
Fixed.
Sharing the v2 patch. Kindly have a look and share your comments.
Thanks & Regards,
Nitin Jadhav
On Tue, Feb 22, 2022 at 12:08 PM Nitin Jadhav
<[email protected]> wrote:
>
> > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > - can be stored without any problem. 'checkpoint or restartpoint?'
> > > (boolean) - can be stored as a integer value like
> > > PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
> > > PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
> > > start time in stat_progress, timestamp fits in 64 bits) - As
> > > Timestamptz is of type int64 internally, so we can store the timestamp
> > > value in the progres parameter and then expose a function like
> > > 'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
> > > Timestamptz) as argument and then returns string representing the
> > > elapsed time.
> >
> > No need to use a string there; I think exposing the checkpoint start
> > time is good enough. The conversion of int64 to timestamp[tz] can be
> > done in SQL (although I'm not sure that exposing the internal bitwise
> > representation of Interval should be exposed to that extent) [0].
> > Users can then extract the duration interval using now() - start_time,
> > which also allows the user to use their own preferred formatting.
>
> The reason for showing the elapsed time rather than exposing the
> timestamp directly is in case of checkpoint during shutdown and
> end-of-recovery, I am planning to log a message in server logs using
> 'log_startup_progress_interval' infrastructure which displays elapsed
> time. So just to match both of the behaviour I am displaying elapsed
> time here. I feel that elapsed time gives a quicker feel of the
> progress. Kindly let me know if you still feel just exposing the
> timestamp is better than showing the elapsed time.
>
> > > 'checkpoint start location' (lsn = uint64) - I feel we
> > > cannot use progress parameters for this case. As assigning uint64 to
> > > int64 type would be an issue for larger values and can lead to hidden
> > > bugs.
> >
> > Not necessarily - we can (without much trouble) do a bitwise cast from
> > uint64 to int64, and then (in SQL) cast it back to a pg_lsn [1]. Not
> > very elegant, but it works quite well.
> >
> > [1] SELECT '0/0'::pg_lsn + ((CASE WHEN stat.my_int64 < 0 THEN
> > pow(2::numeric, 64::numeric)::numeric ELSE 0::numeric END) +
> > stat.my_int64::numeric) FROM (SELECT -2::bigint /* 0xFFFFFFFF/FFFFFFFE
> > */ AS my_bigint_lsn) AS stat(my_int64);
>
> Thanks for sharing. It works. I will include this in the next patch.
> On Sat, Feb 19, 2022 at 11:02 AM Julien Rouhaud <[email protected]> wrote:
> >
> > Hi,
> >
> > On Fri, Feb 18, 2022 at 08:07:05PM +0530, Nitin Jadhav wrote:
> > >
> > > The backend_pid contains a valid value only during
> > > the CHECKPOINT command issued by the backend explicitly, otherwise the
> > > value will be 0. We may have to add an additional field to
> > > 'CheckpointerShmemStruct' to hold the backend pid. The backend
> > > requesting the checkpoint will update its pid to this structure.
> > > Kindly let me know if you still feel the backend_pid field is not
> > > necessary.
> >
> > There are more scenarios where you can have a baackend requesting a checkpoint
> > and waiting for its completion, and there may be more than one backend
> > concerned, so I don't think that storing only one / the first backend pid is
> > ok.
> >
> > > > And also while looking at the patch I see there's the same problem that I
> > > > mentioned in the previous thread, which is that the effective flags can be
> > > > updated once the checkpoint started, and as-is the view won't reflect that. It
> > > > also means that you can't simply display one of wal, time or force but a
> > > > possible combination of the flags (including the one not handled in v1).
> > >
> > > If I understand the above comment properly, it has 2 points. First is
> > > to display the combination of flags rather than just displaying wal,
> > > time or force - The idea behind this is to just let the user know the
> > > reason for checkpointing. That is, the checkpoint is started because
> > > max_wal_size is reached or checkpoint_timeout expired or explicitly
> > > issued CHECKPOINT command. The other flags like CHECKPOINT_IMMEDIATE,
> > > CHECKPOINT_WAIT or CHECKPOINT_FLUSH_ALL indicate how the checkpoint
> > > has to be performed. Hence I have not included those in the view. If
> > > it is really required, I would like to modify the code to include
> > > other flags and display the combination.
> >
> > I think all the information should be exposed. Only knowing why the current
> > checkpoint has been triggered without any further information seems a bit
> > useless. Think for instance for cases like [1].
> >
> > > Second point is to reflect
> > > the updated flags in the view. AFAIK, there is a possibility that the
> > > flags get updated during the on-going checkpoint but the reason for
> > > checkpoint (wal, time or force) will remain same for the current
> > > checkpoint. There might be a change in how checkpoint has to be
> > > performed if CHECKPOINT_IMMEDIATE flag is set. If we go with
> > > displaying the combination of flags in the view, then probably we may
> > > have to reflect this in the view.
> >
> > You can only "upgrade" a checkpoint, but not "downgrade" it. So if for
> > instance you find both CHECKPOINT_CAUSE_TIME and CHECKPOINT_FORCE (which is
> > possible) you can easily know which one was the one that triggered the
> > checkpoint and which one was added later.
> >
> > > > > Probably a new field named 'processes_wiating' or 'events_waiting' can be
> > > > > added for this purpose.
> > > >
> > > > Maybe num_process_waiting?
> > >
> > > I feel 'processes_wiating' aligns more with the naming conventions of
> > > the fields of the existing progres views.
> >
> > There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
> > a strong opinion on it, just make sure to correct the typo.
> >
> > > > > Probably writing of buffers or syncing files may complete before
> > > > > pg_is_in_recovery() returns false. But there are some cleanup
> > > > > operations happen as part of the checkpoint. During this scenario, we
> > > > > may get false value for pg_is_in_recovery(). Please refer following
> > > > > piece of code which is present in CreateRestartpoint().
> > > > >
> > > > > if (!RecoveryInProgress())
> > > > > replayTLI = XLogCtl->InsertTimeLineID;
> > > >
> > > > Then maybe we could store the timeline rather then then kind of checkpoint?
> > > > You should still be able to compute the information while giving a bit more
> > > > information for the same memory usage.
> > >
> > > Can you please describe more about how checkpoint/restartpoint can be
> > > confirmed using the timeline id.
> >
> > If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
> > restartpoint if the checkpoint's timeline is different from the current
> > timeline?
> >
> > [1] https://www.postgresql.org/message-id/1486805889.24568.96.camel%40credativ.de
Attachments:
[application/octet-stream] v2-0001-pg_stat_progress_checkpoint-view.patch (35.0K, ../../CAMm1aWZ07VYUPpvmV9Ck_VX-o+xH=hEBs-67scd_Us6U6FKbHA@mail.gmail.com/2-v2-0001-pg_stat_progress_checkpoint-view.patch)
download | inline diff:
From 9e55ad5e1e1319cc32237e2ba79a097527d07fed Mon Sep 17 00:00:00 2001
From: Nitin Jadhav <[email protected]>
Date: Wed, 23 Feb 2022 13:17:16 +0000
Subject: [PATCH] pg_stat_progress-checkpoint view
---
doc/src/sgml/monitoring.sgml | 357 ++++++++++++++++++++++++++
src/backend/access/transam/xlog.c | 132 ++++++++++
src/backend/catalog/system_views.sql | 36 +++
src/backend/postmaster/checkpointer.c | 14 +-
src/backend/storage/buffer/bufmgr.c | 7 +
src/backend/storage/sync/sync.c | 6 +
src/backend/utils/adt/pgstatfuncs.c | 59 +++++
src/include/access/xlog.h | 4 +
src/include/catalog/pg_proc.dat | 9 +
src/include/commands/progress.h | 28 ++
src/include/utils/backend_progress.h | 3 +-
src/test/regress/expected/rules.out | 33 +++
12 files changed, 684 insertions(+), 4 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index bf7625d988..1e7112eb78 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -401,6 +401,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the <command>CHECKPOINT</command>.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -6895,6 +6902,356 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>CHECKPOINT Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of a CHECKPOINTER process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>kind</structfield> <type>text</type>
+ </para>
+ <para>
+ Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_lsn</structfield> <type>text</type>
+ </para>
+ <para>
+ The checkpoint start location.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>elapsed_time</structfield> <type>text</type>
+ </para>
+ <para>
+ Elapsed time of the checkpoint.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal> total_buffer_writes </literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>total_buffer_writes</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>total_file_syncs</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-types">
+ <title>CHECKPOINT types</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Types</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>checkpoint</literal></entry>
+ <entry>
+ The current operation is checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>restartpoint</literal></entry>
+ <entry>
+ The current operation is restartpoint.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-kinds">
+ <title>CHECKPOINT kinds</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Kinds</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>shutdown</literal></entry>
+ <entry>
+ The checkpoint is for shutdown.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>end-of-recovery</literal></entry>
+ <entry>
+ The checkpoint is for end-of-recovery.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>immediate</literal></entry>
+ <entry>
+ The checkpoint is happens without delays.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint is started because some operation forced a checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>flush all</literal></entry>
+ <entry>
+ The checkpoint flushes all pages, including those belonging to unlogged
+ tables.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wait</literal></entry>
+ <entry>
+ Wait for completion before returning.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>requested</literal></entry>
+ <entry>
+ The checkpoint request has been made.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>max_wal_size</literal> is
+ reached.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>checkpoint_timeout</literal>
+ expired.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>CHECKPOINT phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The CHECKPOINTER process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently flushing all the replication slots
+ to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing snapshots</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently removing all the serialized
+ snapshots that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mappings</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently removing/flushing the logical
+ rewrite mappings.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing CLOG pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing CLOG pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing Commit time stamp pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing Commit time stamp pages to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing SUBTRANS pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing SUBTRANS pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing MULTIXACT pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing MULTIXACT pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing SLRU pages</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing SLRU pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing sync requests</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently performing sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old XLOG files</literal></entry>
+ <entry>
+ The CHECKPOINTER process is currently recycling old XLOG files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The CHECKPOINTER process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 0d2bd7a357..e5a19f3cc3 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -65,6 +65,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "commands/progress.h"
#include "common/controldata_utils.h"
#include "executor/instrument.h"
#include "miscadmin.h"
@@ -653,6 +654,9 @@ static bool updateMinRecoveryPoint = true;
static int MyLockNo = 0;
static bool holdingAllLocks = false;
+/* Copy of checkpoint flags. */
+static int ckpt_flags = 0;
+
#ifdef WAL_DEBUG
static MemoryContext walDebugCxt = NULL;
#endif
@@ -6296,6 +6300,9 @@ CreateCheckPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags);
+
/*
* Use a critical section to force system panic if we have trouble.
*/
@@ -6394,6 +6401,7 @@ CreateCheckPoint(int flags)
curInsert += SizeOfXLogShortPHD;
}
checkPoint.redo = curInsert;
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
/*
* Here we update the shared RedoRecPtr for future XLogInsert calls; this
@@ -6629,8 +6637,12 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/*
* Make more log segments if needed. (Do this after recycling old log
@@ -6652,6 +6664,9 @@ CreateCheckPoint(int flags)
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ checkpoint_progress_end();
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -6808,29 +6823,60 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
CheckPointPredicate();
+
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FILE_SYNC);
ProcessSyncRequests();
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -6977,6 +7023,9 @@ CreateRestartPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the restartpoint. */
+ checkpoint_progress_start(flags);
+
if (log_checkpoints)
LogCheckpointStart(flags, true);
@@ -7077,7 +7126,11 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/*
* Make more log segments if needed. (Do this after recycling old log
@@ -7098,6 +7151,9 @@ CreateRestartPoint(int flags)
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the restartpoint. */
+ checkpoint_progress_end();
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -9197,3 +9253,79 @@ SetWalWriterSleeping(bool sleeping)
XLogCtl->WalWriterSleeping = sleeping;
SpinLockRelease(&XLogCtl->info_lck);
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+void
+checkpoint_progress_start(int flags)
+{
+ /* In bootstrap mode, we don't actually record anything. */
+ if (IsBootstrapProcessingMode())
+ return;
+
+ ckpt_flags = flags;
+
+ /*
+ * Cannot access pg_stat_progress_checkpoint view in case of checkpoint
+ * during shutdown and end-of-recovery.
+ */
+ if ((ckpt_flags &
+ (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
+ {
+ const int index[] = {
+ PROGRESS_CHECKPOINT_TIMELINE,
+ PROGRESS_CHECKPOINT_KIND,
+ PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_START_TIMESTAMP
+ };
+ int64 val[4];
+
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+
+ val[0] = XLogCtl->InsertTimeLineID;
+ val[1] = flags;
+ val[2] = PROGRESS_CHECKPOINT_PHASE_INIT;
+ val[3] = CheckpointStats.ckpt_start_t;
+
+ pgstat_progress_update_multi_param(4, index, val);
+ }
+}
+
+/*
+ * Update index'th member in st_progress_param[] array with the latest value.
+ */
+void
+checkpoint_progress_update_param(int index, int64 val)
+{
+ /* In bootstrap mode, we don't actually record anything. */
+ if (IsBootstrapProcessingMode())
+ return;
+
+ /*
+ * Cannot access pg_stat_progress_checkpoint view in case of checkpoint
+ * during shutdown and end-of-recovery.
+ */
+ if ((ckpt_flags &
+ (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
+ pgstat_progress_update_param(index, val);
+}
+
+/*
+ * Stop reporting progress of the checkpoint.
+ */
+void
+checkpoint_progress_end(void)
+{
+ /* In bootstrap mode, we don't actually record anything. */
+ if (IsBootstrapProcessingMode())
+ return;
+
+ /*
+ * Cannot access pg_stat_progress_checkpoint view in case of checkpoint
+ * during shutdown and end-of-recovery.
+ */
+ if ((ckpt_flags &
+ (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
+ pgstat_progress_end_command();
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 3cb69b1f87..6dc5b0feb6 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1286,3 +1286,39 @@ CREATE VIEW pg_stat_subscription_workers AS
FROM pg_subscription_rel) sr,
LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w
JOIN pg_subscription s ON (w.subid = s.oid);
+
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ pg_stat_get_progress_checkpoint_type(S.param1) AS type,
+ pg_stat_get_progress_checkpoint_kind(S.param2) AS kind,
+ ( SELECT '0/0'::pg_lsn +
+ ((CASE
+ WHEN stat.lsn_int64 < 0 THEN pow(2::numeric, 64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ stat.lsn_int64::numeric)
+ FROM (SELECT s.param3::bigint) AS stat(lsn_int64)
+ ) AS start_lsn,
+ pg_stat_get_progress_checkpoint_elapsed(S.param4) AS elapsed_time,
+ CASE S.param5 WHEN 0 THEN 'initializing'
+ WHEN 1 THEN 'checkpointing replication slots'
+ WHEN 2 THEN 'checkpointing snapshots'
+ WHEN 3 THEN 'checkpointing logical rewrite mappings'
+ WHEN 4 THEN 'checkpointing CLOG pages'
+ WHEN 5 THEN 'checkpointing CommitTs pages'
+ WHEN 6 THEN 'checkpointing SUBTRANS pages'
+ WHEN 7 THEN 'checkpointing MULTIXACT pages'
+ WHEN 8 THEN 'checkpointing SLRU pages'
+ WHEN 9 THEN 'checkpointing buffers'
+ WHEN 10 THEN 'performing sync requests'
+ WHEN 11 THEN 'performing two phase checkpoint'
+ WHEN 12 THEN 'recycling old XLOG files'
+ WHEN 13 THEN 'Finalizing'
+ END AS phase,
+ S.param6 AS total_buffer_writes,
+ S.param7 AS buffers_processed,
+ S.param8 AS buffers_written,
+ S.param9 AS total_file_syncs,
+ S.param10 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 4488e3a443..2ff7e77b3e 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -39,6 +39,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogrecovery.h"
+#include "commands/progress.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -163,7 +164,7 @@ static pg_time_t last_xlog_switch_time;
static void HandleCheckpointerInterrupts(void);
static void CheckArchiveTimeout(void);
static bool IsCheckpointOnSchedule(double progress);
-static bool ImmediateCheckpointRequested(void);
+static bool ImmediateCheckpointRequested(int flags);
static bool CompactCheckpointerRequestQueue(void);
static void UpdateSharedMemoryConfig(void);
@@ -658,16 +659,23 @@ CheckArchiveTimeout(void)
* there is one pending behind it.)
*/
static bool
-ImmediateCheckpointRequested(void)
+ImmediateCheckpointRequested(int flags)
{
volatile CheckpointerShmemStruct *cps = CheckpointerShmem;
+ int updated_flags = flags;
/*
* We don't need to acquire the ckpt_lck in this case because we're only
* looking at a single flag bit.
*/
if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
+ {
+ updated_flags |= CHECKPOINT_IMMEDIATE;
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_KIND,
+ updated_flags);
return true;
+ }
+
return false;
}
@@ -699,7 +707,7 @@ CheckpointWriteDelay(int flags, double progress)
*/
if (!(flags & CHECKPOINT_IMMEDIATE) &&
!ShutdownRequestPending &&
- !ImmediateCheckpointRequested() &&
+ !ImmediateCheckpointRequested(flags) &&
IsCheckpointOnSchedule(progress))
{
if (ConfigReloadPending)
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index f5459c68f8..156130ef43 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -38,6 +38,7 @@
#include "access/xlogutils.h"
#include "catalog/catalog.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2012,6 +2013,8 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_TOTAL,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2129,6 +2132,8 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2149,6 +2154,8 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.m_buf_written_checkpoints++;
num_written++;
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index e161d57761..f0441d4de8 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -356,6 +357,9 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_FILES_TOTAL,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -419,6 +423,8 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ checkpoint_progress_update_param(PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 30e8dfa7c1..32c35f0bd0 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -494,6 +494,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -2495,3 +2497,60 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS)
/* Returns the record as Datum */
PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
}
+
+/*
+ * Return checkpoint type (either checkpoint or restartpoint).
+ */
+Datum
+pg_stat_get_progress_checkpoint_type(PG_FUNCTION_ARGS)
+{
+ TimeLineID cur_timeline = (TimeLineID) PG_GETARG_INT64(0);
+
+ if (RecoveryInProgress() || (GetWALInsertionTimeLine() != cur_timeline))
+ PG_RETURN_TEXT_P(CStringGetTextDatum("restartpoint"));
+ else
+ PG_RETURN_TEXT_P(CStringGetTextDatum("checkpoint"));
+}
+
+/*
+ * Return checkpoint kind based on the flags set.
+ */
+Datum
+pg_stat_get_progress_checkpoint_kind(PG_FUNCTION_ARGS)
+{
+ int64 flags = PG_GETARG_INT64(0);
+ char ckpt_kind[MAXPGPATH];
+
+ MemSet(ckpt_kind, 0, MAXPGPATH);
+ snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
+ (flags == 0) ? "unknown" : "",
+ (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
+ (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
+ (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
+ (flags & CHECKPOINT_FORCE) ? "force " : "",
+ (flags & CHECKPOINT_WAIT) ? "wait " : "",
+ (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
+ (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
+ (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
+
+ PG_RETURN_TEXT_P(CStringGetTextDatum(ckpt_kind));
+}
+
+/*
+ * Return elapsed time (in seconds) of the checkpoint.
+ */
+Datum
+pg_stat_get_progress_checkpoint_elapsed(PG_FUNCTION_ARGS)
+{
+ TimestampTz start = PG_GETARG_INT64(0);
+ TimestampTz now = GetCurrentTimestamp();
+ char elapsed_time[NAMEDATALEN];
+ long secs;
+ int usecs;
+
+ TimestampDifference(start, now, &secs, &usecs);
+ snprintf(elapsed_time, sizeof(elapsed_time), "%ld.%02d s", secs,
+ (usecs / 10000));
+
+ PG_RETURN_TEXT_P(CStringGetTextDatum(elapsed_time));
+}
\ No newline at end of file
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index 4b45ac64db..c0a3f57689 100644
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -288,6 +288,10 @@ extern void do_pg_abort_backup(int code, Datum arg);
extern void register_persistent_abort_backup_handler(void);
extern SessionBackupState get_backup_status(void);
+extern void checkpoint_progress_start(int flags);
+extern void checkpoint_progress_update_param(int index, int64 val);
+extern void checkpoint_progress_end(void);
+
/* File path names (all relative to $PGDATA) */
#define RECOVERY_SIGNAL_FILE "recovery.signal"
#define STANDBY_SIGNAL_FILE "standby.signal"
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7f1ee97f55..40f993c9b1 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5353,6 +5353,15 @@
proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
proargnames => '{cmdtype,pid,datid,relid,param1,param2,param3,param4,param5,param6,param7,param8,param9,param10,param11,param12,param13,param14,param15,param16,param17,param18,param19,param20}',
prosrc => 'pg_stat_get_progress_info' },
+{ oid => '560', descr => 'return checkpoint type',
+ proname => 'pg_stat_get_progress_checkpoint_type', prorettype => 'text',
+ proargtypes => 'int8', prosrc => 'pg_stat_get_progress_checkpoint_type' },
+{ oid => '561', descr => 'return checkpoint kind',
+ proname => 'pg_stat_get_progress_checkpoint_kind', prorettype => 'text',
+ proargtypes => 'int8', prosrc => 'pg_stat_get_progress_checkpoint_kind' },
+{ oid => '562', descr => 'return elapsed time of the checkpoint',
+ proname => 'pg_stat_get_progress_checkpoint_elapsed', prorettype => 'text',
+ proargtypes => 'int8', prosrc => 'pg_stat_get_progress_checkpoint_elapsed' },
{ oid => '3099',
descr => 'statistics: information about currently active replication',
proname => 'pg_stat_get_wal_senders', prorows => '10', proisstrict => 'f',
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..19734405ed 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,32 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_TIMELINE 0
+#define PROGRESS_CHECKPOINT_KIND 1
+#define PROGRESS_CHECKPOINT_LSN 2
+#define PROGRESS_CHECKPOINT_START_TIMESTAMP 3
+#define PROGRESS_CHECKPOINT_PHASE 4
+#define PROGRESS_CHECKPOINT_BUFFERS_TOTAL 5
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 6
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 7
+#define PROGRESS_CHECKPOINT_FILES_TOTAL 8
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 9
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 0
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 1
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 2
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 3
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 4
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 5
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 6
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 9
+#define PROGRESS_CHECKPOINT_PHASE_FILE_SYNC 10
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 11
+#define PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE 12
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 13
+
#endif
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 1420288d67..fe6e31ca27 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1897,6 +1897,39 @@ pg_stat_progress_basebackup| SELECT s.pid,
s.param4 AS tablespaces_total,
s.param5 AS tablespaces_streamed
FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
+pg_stat_progress_checkpoint| SELECT s.pid,
+ pg_stat_get_progress_checkpoint_type(s.param1) AS type,
+ pg_stat_get_progress_checkpoint_kind(s.param2) AS kind,
+ ( SELECT ('0/0'::pg_lsn + (
+ CASE
+ WHEN (stat.lsn_int64 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (stat.lsn_int64)::numeric))
+ FROM ( SELECT s.param3) stat(lsn_int64)) AS start_lsn,
+ pg_stat_get_progress_checkpoint_elapsed(s.param4) AS elapsed_time,
+ CASE s.param5
+ WHEN 0 THEN 'initializing'::text
+ WHEN 1 THEN 'checkpointing replication slots'::text
+ WHEN 2 THEN 'checkpointing snapshots'::text
+ WHEN 3 THEN 'checkpointing logical rewrite mappings'::text
+ WHEN 4 THEN 'checkpointing CLOG pages'::text
+ WHEN 5 THEN 'checkpointing CommitTs pages'::text
+ WHEN 6 THEN 'checkpointing SUBTRANS pages'::text
+ WHEN 7 THEN 'checkpointing MULTIXACT pages'::text
+ WHEN 8 THEN 'checkpointing SLRU pages'::text
+ WHEN 9 THEN 'checkpointing buffers'::text
+ WHEN 10 THEN 'performing sync requests'::text
+ WHEN 11 THEN 'performing two phase checkpoint'::text
+ WHEN 12 THEN 'recycling old XLOG files'::text
+ WHEN 13 THEN 'Finalizing'::text
+ ELSE NULL::text
+ END AS phase,
+ s.param6 AS total_buffer_writes,
+ s.param7 AS buffers_processed,
+ s.param8 AS buffers_written,
+ s.param9 AS total_file_syncs,
+ s.param10 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
pg_stat_progress_cluster| SELECT s.pid,
s.datid,
d.datname,
--
2.25.1
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-23 16:16 ` Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
3 siblings, 1 reply; 85+ messages in thread
From: Ashutosh Sharma @ 2022-02-23 16:16 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
+ if ((ckpt_flags &
+ (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
+ {
This code (present at multiple places) looks a little ugly to me, what
we can do instead is add a macro probably named IsShutdownCheckpoint()
which does the above check and use it in all the functions that have
this check. See below:
#define IsShutdownCheckpoint(flags) \
(flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY) != 0)
And then you may use this macro like:
if (IsBootstrapProcessingMode() || IsShutdownCheckpoint(flags))
return;
This change can be done in all these functions:
+void
+checkpoint_progress_start(int flags)
--
+ */
+void
+checkpoint_progress_update_param(int index, int64 val)
--
+ * Stop reporting progress of the checkpoint.
+ */
+void
+checkpoint_progress_end(void)
==
+
pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT,
InvalidOid);
+
+ val[0] = XLogCtl->InsertTimeLineID;
+ val[1] = flags;
+ val[2] = PROGRESS_CHECKPOINT_PHASE_INIT;
+ val[3] = CheckpointStats.ckpt_start_t;
+
+ pgstat_progress_update_multi_param(4, index, val);
+ }
Any specific reason for recording the timelineID in checkpoint stats
table? Will this ever change in our case?
--
With Regards,
Ashutosh Sharma.
On Wed, Feb 23, 2022 at 6:59 PM Nitin Jadhav
<[email protected]> wrote:
>
> > I will make use of pgstat_progress_update_multi_param() in the next
> > patch to replace multiple calls to checkpoint_progress_update_param().
>
> Fixed.
> ---
>
> > > The other progress tables use [type]_total as column names for counter
> > > targets (e.g. backup_total for backup_streamed, heap_blks_total for
> > > heap_blks_scanned, etc.). I think that `buffers_total` and
> > > `files_total` would be better column names.
> >
> > I agree and I will update this in the next patch.
>
> Fixed.
> ---
>
> > How about this "The checkpoint is started because max_wal_size is reached".
> >
> > "The checkpoint is started because checkpoint_timeout expired".
> >
> > "The checkpoint is started because some operation forced a checkpoint".
>
> I have used the above description. Kindly let me know if any changes
> are required.
> ---
>
> > > > + <entry><literal>checkpointing CommitTs pages</literal></entry>
> > >
> > > CommitTs -> Commit time stamp
> >
> > I will handle this in the next patch.
>
> Fixed.
> ---
>
> > There are more scenarios where you can have a baackend requesting a checkpoint
> > and waiting for its completion, and there may be more than one backend
> > concerned, so I don't think that storing only one / the first backend pid is
> > ok.
>
> Thanks for this information. I am not considering backend_pid.
> ---
>
> > I think all the information should be exposed. Only knowing why the current
> > checkpoint has been triggered without any further information seems a bit
> > useless. Think for instance for cases like [1].
>
> I have supported all possible checkpoint kinds. Added
> pg_stat_get_progress_checkpoint_kind() to convert the flags (int) to a
> string representing a combination of flags and also checking for the
> flag update in ImmediateCheckpointRequested() which checks whether
> CHECKPOINT_IMMEDIATE flag is set or not. I did not find any other
> cases where the flags get changed (which changes the current
> checkpoint behaviour) during the checkpoint. Kindly let me know if I
> am missing something.
> ---
>
> > > I feel 'processes_wiating' aligns more with the naming conventions of
> > > the fields of the existing progres views.
> >
> > There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
> > a strong opinion on it, just make sure to correct the typo.
>
> More analysis is required to support this. I am planning to take care
> in the next patch.
> ---
>
> > If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
> > restartpoint if the checkpoint's timeline is different from the current
> > timeline?
>
> Fixed.
>
> Sharing the v2 patch. Kindly have a look and share your comments.
>
> Thanks & Regards,
> Nitin Jadhav
>
>
>
>
> On Tue, Feb 22, 2022 at 12:08 PM Nitin Jadhav
> <[email protected]> wrote:
> >
> > > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > > - can be stored without any problem. 'checkpoint or restartpoint?'
> > > > (boolean) - can be stored as a integer value like
> > > > PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
> > > > PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
> > > > start time in stat_progress, timestamp fits in 64 bits) - As
> > > > Timestamptz is of type int64 internally, so we can store the timestamp
> > > > value in the progres parameter and then expose a function like
> > > > 'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
> > > > Timestamptz) as argument and then returns string representing the
> > > > elapsed time.
> > >
> > > No need to use a string there; I think exposing the checkpoint start
> > > time is good enough. The conversion of int64 to timestamp[tz] can be
> > > done in SQL (although I'm not sure that exposing the internal bitwise
> > > representation of Interval should be exposed to that extent) [0].
> > > Users can then extract the duration interval using now() - start_time,
> > > which also allows the user to use their own preferred formatting.
> >
> > The reason for showing the elapsed time rather than exposing the
> > timestamp directly is in case of checkpoint during shutdown and
> > end-of-recovery, I am planning to log a message in server logs using
> > 'log_startup_progress_interval' infrastructure which displays elapsed
> > time. So just to match both of the behaviour I am displaying elapsed
> > time here. I feel that elapsed time gives a quicker feel of the
> > progress. Kindly let me know if you still feel just exposing the
> > timestamp is better than showing the elapsed time.
> >
> > > > 'checkpoint start location' (lsn = uint64) - I feel we
> > > > cannot use progress parameters for this case. As assigning uint64 to
> > > > int64 type would be an issue for larger values and can lead to hidden
> > > > bugs.
> > >
> > > Not necessarily - we can (without much trouble) do a bitwise cast from
> > > uint64 to int64, and then (in SQL) cast it back to a pg_lsn [1]. Not
> > > very elegant, but it works quite well.
> > >
> > > [1] SELECT '0/0'::pg_lsn + ((CASE WHEN stat.my_int64 < 0 THEN
> > > pow(2::numeric, 64::numeric)::numeric ELSE 0::numeric END) +
> > > stat.my_int64::numeric) FROM (SELECT -2::bigint /* 0xFFFFFFFF/FFFFFFFE
> > > */ AS my_bigint_lsn) AS stat(my_int64);
> >
> > Thanks for sharing. It works. I will include this in the next patch.
> > On Sat, Feb 19, 2022 at 11:02 AM Julien Rouhaud <[email protected]> wrote:
> > >
> > > Hi,
> > >
> > > On Fri, Feb 18, 2022 at 08:07:05PM +0530, Nitin Jadhav wrote:
> > > >
> > > > The backend_pid contains a valid value only during
> > > > the CHECKPOINT command issued by the backend explicitly, otherwise the
> > > > value will be 0. We may have to add an additional field to
> > > > 'CheckpointerShmemStruct' to hold the backend pid. The backend
> > > > requesting the checkpoint will update its pid to this structure.
> > > > Kindly let me know if you still feel the backend_pid field is not
> > > > necessary.
> > >
> > > There are more scenarios where you can have a baackend requesting a checkpoint
> > > and waiting for its completion, and there may be more than one backend
> > > concerned, so I don't think that storing only one / the first backend pid is
> > > ok.
> > >
> > > > > And also while looking at the patch I see there's the same problem that I
> > > > > mentioned in the previous thread, which is that the effective flags can be
> > > > > updated once the checkpoint started, and as-is the view won't reflect that. It
> > > > > also means that you can't simply display one of wal, time or force but a
> > > > > possible combination of the flags (including the one not handled in v1).
> > > >
> > > > If I understand the above comment properly, it has 2 points. First is
> > > > to display the combination of flags rather than just displaying wal,
> > > > time or force - The idea behind this is to just let the user know the
> > > > reason for checkpointing. That is, the checkpoint is started because
> > > > max_wal_size is reached or checkpoint_timeout expired or explicitly
> > > > issued CHECKPOINT command. The other flags like CHECKPOINT_IMMEDIATE,
> > > > CHECKPOINT_WAIT or CHECKPOINT_FLUSH_ALL indicate how the checkpoint
> > > > has to be performed. Hence I have not included those in the view. If
> > > > it is really required, I would like to modify the code to include
> > > > other flags and display the combination.
> > >
> > > I think all the information should be exposed. Only knowing why the current
> > > checkpoint has been triggered without any further information seems a bit
> > > useless. Think for instance for cases like [1].
> > >
> > > > Second point is to reflect
> > > > the updated flags in the view. AFAIK, there is a possibility that the
> > > > flags get updated during the on-going checkpoint but the reason for
> > > > checkpoint (wal, time or force) will remain same for the current
> > > > checkpoint. There might be a change in how checkpoint has to be
> > > > performed if CHECKPOINT_IMMEDIATE flag is set. If we go with
> > > > displaying the combination of flags in the view, then probably we may
> > > > have to reflect this in the view.
> > >
> > > You can only "upgrade" a checkpoint, but not "downgrade" it. So if for
> > > instance you find both CHECKPOINT_CAUSE_TIME and CHECKPOINT_FORCE (which is
> > > possible) you can easily know which one was the one that triggered the
> > > checkpoint and which one was added later.
> > >
> > > > > > Probably a new field named 'processes_wiating' or 'events_waiting' can be
> > > > > > added for this purpose.
> > > > >
> > > > > Maybe num_process_waiting?
> > > >
> > > > I feel 'processes_wiating' aligns more with the naming conventions of
> > > > the fields of the existing progres views.
> > >
> > > There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
> > > a strong opinion on it, just make sure to correct the typo.
> > >
> > > > > > Probably writing of buffers or syncing files may complete before
> > > > > > pg_is_in_recovery() returns false. But there are some cleanup
> > > > > > operations happen as part of the checkpoint. During this scenario, we
> > > > > > may get false value for pg_is_in_recovery(). Please refer following
> > > > > > piece of code which is present in CreateRestartpoint().
> > > > > >
> > > > > > if (!RecoveryInProgress())
> > > > > > replayTLI = XLogCtl->InsertTimeLineID;
> > > > >
> > > > > Then maybe we could store the timeline rather then then kind of checkpoint?
> > > > > You should still be able to compute the information while giving a bit more
> > > > > information for the same memory usage.
> > > >
> > > > Can you please describe more about how checkpoint/restartpoint can be
> > > > confirmed using the timeline id.
> > >
> > > If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
> > > restartpoint if the checkpoint's timeline is different from the current
> > > timeline?
> > >
> > > [1] https://www.postgresql.org/message-id/1486805889.24568.96.camel%40credativ.de
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
@ 2022-02-25 15:07 ` Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-25 15:07 UTC (permalink / raw)
To: Ashutosh Sharma <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> + if ((ckpt_flags &
> + (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
> + {
>
> This code (present at multiple places) looks a little ugly to me, what
> we can do instead is add a macro probably named IsShutdownCheckpoint()
> which does the above check and use it in all the functions that have
> this check. See below:
>
> #define IsShutdownCheckpoint(flags) \
> (flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY) != 0)
>
> And then you may use this macro like:
>
> if (IsBootstrapProcessingMode() || IsShutdownCheckpoint(flags))
> return;
Good suggestion. In the v3 patch, I have removed the corresponding
code as these checks are not required. Hence this suggestion is not
applicable now.
---
> pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT,
> InvalidOid);
> +
> + val[0] = XLogCtl->InsertTimeLineID;
> + val[1] = flags;
> + val[2] = PROGRESS_CHECKPOINT_PHASE_INIT;
> + val[3] = CheckpointStats.ckpt_start_t;
> +
> + pgstat_progress_update_multi_param(4, index, val);
> + }
>
> Any specific reason for recording the timelineID in checkpoint stats
> table? Will this ever change in our case?
The timelineID is used to decide whether the current operation is
checkpoint or restartpoint. There is a field in the view to display
this information.
Thanks & Regards,
Nitin Jadhav
On Wed, Feb 23, 2022 at 9:46 PM Ashutosh Sharma <[email protected]> wrote:
>
> + if ((ckpt_flags &
> + (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) == 0)
> + {
>
> This code (present at multiple places) looks a little ugly to me, what
> we can do instead is add a macro probably named IsShutdownCheckpoint()
> which does the above check and use it in all the functions that have
> this check. See below:
>
> #define IsShutdownCheckpoint(flags) \
> (flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY) != 0)
>
> And then you may use this macro like:
>
> if (IsBootstrapProcessingMode() || IsShutdownCheckpoint(flags))
> return;
>
> This change can be done in all these functions:
>
> +void
> +checkpoint_progress_start(int flags)
>
> --
>
> + */
> +void
> +checkpoint_progress_update_param(int index, int64 val)
>
> --
>
> + * Stop reporting progress of the checkpoint.
> + */
> +void
> +checkpoint_progress_end(void)
>
> ==
>
> +
> pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT,
> InvalidOid);
> +
> + val[0] = XLogCtl->InsertTimeLineID;
> + val[1] = flags;
> + val[2] = PROGRESS_CHECKPOINT_PHASE_INIT;
> + val[3] = CheckpointStats.ckpt_start_t;
> +
> + pgstat_progress_update_multi_param(4, index, val);
> + }
>
> Any specific reason for recording the timelineID in checkpoint stats
> table? Will this ever change in our case?
>
> --
> With Regards,
> Ashutosh Sharma.
>
> On Wed, Feb 23, 2022 at 6:59 PM Nitin Jadhav
> <[email protected]> wrote:
> >
> > > I will make use of pgstat_progress_update_multi_param() in the next
> > > patch to replace multiple calls to checkpoint_progress_update_param().
> >
> > Fixed.
> > ---
> >
> > > > The other progress tables use [type]_total as column names for counter
> > > > targets (e.g. backup_total for backup_streamed, heap_blks_total for
> > > > heap_blks_scanned, etc.). I think that `buffers_total` and
> > > > `files_total` would be better column names.
> > >
> > > I agree and I will update this in the next patch.
> >
> > Fixed.
> > ---
> >
> > > How about this "The checkpoint is started because max_wal_size is reached".
> > >
> > > "The checkpoint is started because checkpoint_timeout expired".
> > >
> > > "The checkpoint is started because some operation forced a checkpoint".
> >
> > I have used the above description. Kindly let me know if any changes
> > are required.
> > ---
> >
> > > > > + <entry><literal>checkpointing CommitTs pages</literal></entry>
> > > >
> > > > CommitTs -> Commit time stamp
> > >
> > > I will handle this in the next patch.
> >
> > Fixed.
> > ---
> >
> > > There are more scenarios where you can have a baackend requesting a checkpoint
> > > and waiting for its completion, and there may be more than one backend
> > > concerned, so I don't think that storing only one / the first backend pid is
> > > ok.
> >
> > Thanks for this information. I am not considering backend_pid.
> > ---
> >
> > > I think all the information should be exposed. Only knowing why the current
> > > checkpoint has been triggered without any further information seems a bit
> > > useless. Think for instance for cases like [1].
> >
> > I have supported all possible checkpoint kinds. Added
> > pg_stat_get_progress_checkpoint_kind() to convert the flags (int) to a
> > string representing a combination of flags and also checking for the
> > flag update in ImmediateCheckpointRequested() which checks whether
> > CHECKPOINT_IMMEDIATE flag is set or not. I did not find any other
> > cases where the flags get changed (which changes the current
> > checkpoint behaviour) during the checkpoint. Kindly let me know if I
> > am missing something.
> > ---
> >
> > > > I feel 'processes_wiating' aligns more with the naming conventions of
> > > > the fields of the existing progres views.
> > >
> > > There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
> > > a strong opinion on it, just make sure to correct the typo.
> >
> > More analysis is required to support this. I am planning to take care
> > in the next patch.
> > ---
> >
> > > If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
> > > restartpoint if the checkpoint's timeline is different from the current
> > > timeline?
> >
> > Fixed.
> >
> > Sharing the v2 patch. Kindly have a look and share your comments.
> >
> > Thanks & Regards,
> > Nitin Jadhav
> >
> >
> >
> >
> > On Tue, Feb 22, 2022 at 12:08 PM Nitin Jadhav
> > <[email protected]> wrote:
> > >
> > > > > Thank you for sharing the information. 'triggering backend PID' (int)
> > > > > - can be stored without any problem. 'checkpoint or restartpoint?'
> > > > > (boolean) - can be stored as a integer value like
> > > > > PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
> > > > > PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
> > > > > start time in stat_progress, timestamp fits in 64 bits) - As
> > > > > Timestamptz is of type int64 internally, so we can store the timestamp
> > > > > value in the progres parameter and then expose a function like
> > > > > 'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
> > > > > Timestamptz) as argument and then returns string representing the
> > > > > elapsed time.
> > > >
> > > > No need to use a string there; I think exposing the checkpoint start
> > > > time is good enough. The conversion of int64 to timestamp[tz] can be
> > > > done in SQL (although I'm not sure that exposing the internal bitwise
> > > > representation of Interval should be exposed to that extent) [0].
> > > > Users can then extract the duration interval using now() - start_time,
> > > > which also allows the user to use their own preferred formatting.
> > >
> > > The reason for showing the elapsed time rather than exposing the
> > > timestamp directly is in case of checkpoint during shutdown and
> > > end-of-recovery, I am planning to log a message in server logs using
> > > 'log_startup_progress_interval' infrastructure which displays elapsed
> > > time. So just to match both of the behaviour I am displaying elapsed
> > > time here. I feel that elapsed time gives a quicker feel of the
> > > progress. Kindly let me know if you still feel just exposing the
> > > timestamp is better than showing the elapsed time.
> > >
> > > > > 'checkpoint start location' (lsn = uint64) - I feel we
> > > > > cannot use progress parameters for this case. As assigning uint64 to
> > > > > int64 type would be an issue for larger values and can lead to hidden
> > > > > bugs.
> > > >
> > > > Not necessarily - we can (without much trouble) do a bitwise cast from
> > > > uint64 to int64, and then (in SQL) cast it back to a pg_lsn [1]. Not
> > > > very elegant, but it works quite well.
> > > >
> > > > [1] SELECT '0/0'::pg_lsn + ((CASE WHEN stat.my_int64 < 0 THEN
> > > > pow(2::numeric, 64::numeric)::numeric ELSE 0::numeric END) +
> > > > stat.my_int64::numeric) FROM (SELECT -2::bigint /* 0xFFFFFFFF/FFFFFFFE
> > > > */ AS my_bigint_lsn) AS stat(my_int64);
> > >
> > > Thanks for sharing. It works. I will include this in the next patch.
> > > On Sat, Feb 19, 2022 at 11:02 AM Julien Rouhaud <[email protected]> wrote:
> > > >
> > > > Hi,
> > > >
> > > > On Fri, Feb 18, 2022 at 08:07:05PM +0530, Nitin Jadhav wrote:
> > > > >
> > > > > The backend_pid contains a valid value only during
> > > > > the CHECKPOINT command issued by the backend explicitly, otherwise the
> > > > > value will be 0. We may have to add an additional field to
> > > > > 'CheckpointerShmemStruct' to hold the backend pid. The backend
> > > > > requesting the checkpoint will update its pid to this structure.
> > > > > Kindly let me know if you still feel the backend_pid field is not
> > > > > necessary.
> > > >
> > > > There are more scenarios where you can have a baackend requesting a checkpoint
> > > > and waiting for its completion, and there may be more than one backend
> > > > concerned, so I don't think that storing only one / the first backend pid is
> > > > ok.
> > > >
> > > > > > And also while looking at the patch I see there's the same problem that I
> > > > > > mentioned in the previous thread, which is that the effective flags can be
> > > > > > updated once the checkpoint started, and as-is the view won't reflect that. It
> > > > > > also means that you can't simply display one of wal, time or force but a
> > > > > > possible combination of the flags (including the one not handled in v1).
> > > > >
> > > > > If I understand the above comment properly, it has 2 points. First is
> > > > > to display the combination of flags rather than just displaying wal,
> > > > > time or force - The idea behind this is to just let the user know the
> > > > > reason for checkpointing. That is, the checkpoint is started because
> > > > > max_wal_size is reached or checkpoint_timeout expired or explicitly
> > > > > issued CHECKPOINT command. The other flags like CHECKPOINT_IMMEDIATE,
> > > > > CHECKPOINT_WAIT or CHECKPOINT_FLUSH_ALL indicate how the checkpoint
> > > > > has to be performed. Hence I have not included those in the view. If
> > > > > it is really required, I would like to modify the code to include
> > > > > other flags and display the combination.
> > > >
> > > > I think all the information should be exposed. Only knowing why the current
> > > > checkpoint has been triggered without any further information seems a bit
> > > > useless. Think for instance for cases like [1].
> > > >
> > > > > Second point is to reflect
> > > > > the updated flags in the view. AFAIK, there is a possibility that the
> > > > > flags get updated during the on-going checkpoint but the reason for
> > > > > checkpoint (wal, time or force) will remain same for the current
> > > > > checkpoint. There might be a change in how checkpoint has to be
> > > > > performed if CHECKPOINT_IMMEDIATE flag is set. If we go with
> > > > > displaying the combination of flags in the view, then probably we may
> > > > > have to reflect this in the view.
> > > >
> > > > You can only "upgrade" a checkpoint, but not "downgrade" it. So if for
> > > > instance you find both CHECKPOINT_CAUSE_TIME and CHECKPOINT_FORCE (which is
> > > > possible) you can easily know which one was the one that triggered the
> > > > checkpoint and which one was added later.
> > > >
> > > > > > > Probably a new field named 'processes_wiating' or 'events_waiting' can be
> > > > > > > added for this purpose.
> > > > > >
> > > > > > Maybe num_process_waiting?
> > > > >
> > > > > I feel 'processes_wiating' aligns more with the naming conventions of
> > > > > the fields of the existing progres views.
> > > >
> > > > There's at least pg_stat_progress_vacuum.num_dead_tuples. Anyway I don't have
> > > > a strong opinion on it, just make sure to correct the typo.
> > > >
> > > > > > > Probably writing of buffers or syncing files may complete before
> > > > > > > pg_is_in_recovery() returns false. But there are some cleanup
> > > > > > > operations happen as part of the checkpoint. During this scenario, we
> > > > > > > may get false value for pg_is_in_recovery(). Please refer following
> > > > > > > piece of code which is present in CreateRestartpoint().
> > > > > > >
> > > > > > > if (!RecoveryInProgress())
> > > > > > > replayTLI = XLogCtl->InsertTimeLineID;
> > > > > >
> > > > > > Then maybe we could store the timeline rather then then kind of checkpoint?
> > > > > > You should still be able to compute the information while giving a bit more
> > > > > > information for the same memory usage.
> > > > >
> > > > > Can you please describe more about how checkpoint/restartpoint can be
> > > > > confirmed using the timeline id.
> > > >
> > > > If pg_is_in_recovery() is true, then it's a restartpoint, otherwise it's a
> > > > restartpoint if the checkpoint's timeline is different from the current
> > > > timeline?
> > > >
> > > > [1] https://www.postgresql.org/message-id/1486805889.24568.96.camel%40credativ.de
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-27 15:14 ` Bharath Rupireddy <[email protected]>
2022-02-28 04:51 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
0 siblings, 2 replies; 85+ messages in thread
From: Bharath Rupireddy @ 2022-02-27 15:14 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Feb 25, 2022 at 8:38 PM Nitin Jadhav
<[email protected]> wrote:
Had a quick look over the v3 patch. I'm not sure if it's the best way
to have pg_stat_get_progress_checkpoint_type,
pg_stat_get_progress_checkpoint_kind and
pg_stat_get_progress_checkpoint_start_time just for printing info in
readable format in pg_stat_progress_checkpoint. I don't think these
functions will ever be useful for the users.
1) Can't we use pg_is_in_recovery to determine if it's a restartpoint
or checkpoint instead of having a new function
pg_stat_get_progress_checkpoint_type?
2) Can't we just have these checks inside CASE-WHEN-THEN-ELSE blocks
directly instead of new function pg_stat_get_progress_checkpoint_kind?
+ snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
+ (flags == 0) ? "unknown" : "",
+ (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
+ (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
+ (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
+ (flags & CHECKPOINT_FORCE) ? "force " : "",
+ (flags & CHECKPOINT_WAIT) ? "wait " : "",
+ (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
+ (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
+ (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
3) Why do we need this extra calculation for start_lsn? Do you ever
see a negative LSN or something here?
+ ('0/0'::pg_lsn + (
+ CASE
+ WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (s.param3)::numeric)) AS start_lsn,
4) Can't you use timestamptz_in(to_char(s.param4)) instead of
pg_stat_get_progress_checkpoint_start_time? I don't quite understand
the reasoning for having this function and it's named as *checkpoint*
when it doesn't do anything specific to the checkpoint at all?
Having 3 unnecessary functions that aren't useful to the users at all
in proc.dat will simply eatup the function oids IMO. Hence, I suggest
let's try to do without extra functions.
Regards,
Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
@ 2022-02-28 04:51 ` Bharath Rupireddy <[email protected]>
2022-02-28 06:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
1 sibling, 1 reply; 85+ messages in thread
From: Bharath Rupireddy @ 2022-02-28 04:51 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sun, Feb 27, 2022 at 8:44 PM Bharath Rupireddy
<[email protected]> wrote:
>
> On Fri, Feb 25, 2022 at 8:38 PM Nitin Jadhav
> <[email protected]> wrote:
>
> Had a quick look over the v3 patch. I'm not sure if it's the best way
> to have pg_stat_get_progress_checkpoint_type,
> pg_stat_get_progress_checkpoint_kind and
> pg_stat_get_progress_checkpoint_start_time just for printing info in
> readable format in pg_stat_progress_checkpoint. I don't think these
> functions will ever be useful for the users.
>
> 1) Can't we use pg_is_in_recovery to determine if it's a restartpoint
> or checkpoint instead of having a new function
> pg_stat_get_progress_checkpoint_type?
>
> 2) Can't we just have these checks inside CASE-WHEN-THEN-ELSE blocks
> directly instead of new function pg_stat_get_progress_checkpoint_kind?
> + snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
> + (flags == 0) ? "unknown" : "",
> + (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
> + (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
> + (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
> + (flags & CHECKPOINT_FORCE) ? "force " : "",
> + (flags & CHECKPOINT_WAIT) ? "wait " : "",
> + (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
> + (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
> + (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
>
> 3) Why do we need this extra calculation for start_lsn? Do you ever
> see a negative LSN or something here?
> + ('0/0'::pg_lsn + (
> + CASE
> + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> + ELSE (0)::numeric
> + END + (s.param3)::numeric)) AS start_lsn,
>
> 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> the reasoning for having this function and it's named as *checkpoint*
> when it doesn't do anything specific to the checkpoint at all?
>
> Having 3 unnecessary functions that aren't useful to the users at all
> in proc.dat will simply eatup the function oids IMO. Hence, I suggest
> let's try to do without extra functions.
Another thought for my review comment:
> 1) Can't we use pg_is_in_recovery to determine if it's a restartpoint
> or checkpoint instead of having a new function
> pg_stat_get_progress_checkpoint_type?
I don't think using pg_is_in_recovery work here as it is taken after
the checkpoint has started. So, I think the right way here is to send
1 in CreateCheckPoint and 2 in CreateRestartPoint and use
CASE-WHEN-ELSE-END to show "1": "checkpoint" "2":"restartpoint".
Continuing my review:
5) Do we need a special phase for this checkpoint operation? I'm not
sure in which cases it will take a long time, but it looks like
there's a wait loop here.
vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
if (nvxids > 0)
{
do
{
pg_usleep(10000L); /* wait for 10 msec */
} while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids));
}
Also, how about special phases for SyncPostCheckpoint(),
SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
it might be increase in future (?)), TruncateSUBTRANS()?
6) SLRU (Simple LRU) isn't a phase here, you can just say
PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES.
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
CheckPointPredicate();
And :s/checkpointing SLRU pages/checkpointing predicate lock pages
+ WHEN 9 THEN 'checkpointing SLRU pages'
7) :s/PROGRESS_CHECKPOINT_PHASE_FILE_SYNC/PROGRESS_CHECKPOINT_PHASE_PROCESS_FILE_SYNC_REQUESTS
And :s/WHEN 11 THEN 'performing sync requests'/WHEN 11 THEN
'processing file sync requests'
8) :s/Finalizing/finalizing
+ WHEN 14 THEN 'Finalizing'
9) :s/checkpointing snapshots/checkpointing logical replication snapshot files
+ WHEN 3 THEN 'checkpointing snapshots'
:s/checkpointing logical rewrite mappings/checkpointing logical
replication rewrite mapping files
+ WHEN 4 THEN 'checkpointing logical rewrite mappings'
10) I'm not sure if it's discussed, how about adding the number of
snapshot/mapping files so far the checkpoint has processed in file
processing while loops of
CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
be many logical snapshot or mapping files and users may be interested
in knowing the so-far-processed-file-count.
11) I think it's discussed, are we going to add the pid of the
checkpoint requestor?
Regards,
Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 04:51 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
@ 2022-02-28 06:32 ` Julien Rouhaud <[email protected]>
2022-02-28 12:33 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Julien Rouhaud @ 2022-02-28 06:32 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Ashutosh Sharma <[email protected]>; Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On Mon, Feb 28, 2022 at 10:21:23AM +0530, Bharath Rupireddy wrote:
>
> Another thought for my review comment:
> > 1) Can't we use pg_is_in_recovery to determine if it's a restartpoint
> > or checkpoint instead of having a new function
> > pg_stat_get_progress_checkpoint_type?
>
> I don't think using pg_is_in_recovery work here as it is taken after
> the checkpoint has started. So, I think the right way here is to send
> 1 in CreateCheckPoint and 2 in CreateRestartPoint and use
> CASE-WHEN-ELSE-END to show "1": "checkpoint" "2":"restartpoint".
I suggested upthread to store the starting timeline instead. This way you can
deduce whether it's a restartpoint or a checkpoint, but you can also deduce
other information, like what was the starting WAL.
> 11) I think it's discussed, are we going to add the pid of the
> checkpoint requestor?
As mentioned upthread, there can be multiple backends that request a
checkpoint, so unless we want to store an array of pid we should store a number
of backend that are waiting for a new checkpoint.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 04:51 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 06:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-02-28 12:33 ` Bharath Rupireddy <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Bharath Rupireddy @ 2022-02-28 12:33 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Ashutosh Sharma <[email protected]>; Matthias van de Meent <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Mon, Feb 28, 2022 at 12:02 PM Julien Rouhaud <[email protected]> wrote:
>
> Hi,
>
> On Mon, Feb 28, 2022 at 10:21:23AM +0530, Bharath Rupireddy wrote:
> >
> > Another thought for my review comment:
> > > 1) Can't we use pg_is_in_recovery to determine if it's a restartpoint
> > > or checkpoint instead of having a new function
> > > pg_stat_get_progress_checkpoint_type?
> >
> > I don't think using pg_is_in_recovery work here as it is taken after
> > the checkpoint has started. So, I think the right way here is to send
> > 1 in CreateCheckPoint and 2 in CreateRestartPoint and use
> > CASE-WHEN-ELSE-END to show "1": "checkpoint" "2":"restartpoint".
>
> I suggested upthread to store the starting timeline instead. This way you can
> deduce whether it's a restartpoint or a checkpoint, but you can also deduce
> other information, like what was the starting WAL.
I don't understand why we need the timeline here to just determine
whether it's a restartpoint or checkpoint. I know that the
InsertTimeLineID is 0 during recovery. IMO, emitting 1 for checkpoint
and 2 for restartpoint in CreateCheckPoint and CreateRestartPoint
respectively and using CASE-WHEN-ELSE-END to show it in readable
format is the easiest way.
Can't the checkpoint start LSN be deduced from
PROGRESS_CHECKPOINT_LSN, checkPoint.redo?
I'm completely against these pg_stat_get_progress_checkpoint_{type,
kind, start_time} functions unless there's a strong case. IMO, we can
achieve what we want without these functions as well.
> > 11) I think it's discussed, are we going to add the pid of the
> > checkpoint requestor?
>
> As mentioned upthread, there can be multiple backends that request a
> checkpoint, so unless we want to store an array of pid we should store a number
> of backend that are waiting for a new checkpoint.
Yeah, you are right. Let's not go that path and store an array of
pids. I don't see a strong use-case with the pid of the process
requesting checkpoint. If required, we can add it later once the
pg_stat_progress_checkpoint view gets in.
Regards,
Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
@ 2022-02-28 13:10 ` Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
1 sibling, 1 reply; 85+ messages in thread
From: Matthias van de Meent @ 2022-02-28 13:10 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sun, 27 Feb 2022 at 16:14, Bharath Rupireddy
<[email protected]> wrote:
> 3) Why do we need this extra calculation for start_lsn?
> Do you ever see a negative LSN or something here?
> + ('0/0'::pg_lsn + (
> + CASE
> + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> + ELSE (0)::numeric
> + END + (s.param3)::numeric)) AS start_lsn,
Yes: LSN can take up all of an uint64; whereas the pgstat column is a
bigint type; thus the signed int64. This cast is OK as it wraps
around, but that means we have to take care to correctly display the
LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
the special-casing for negative values.
As to whether it is reasonable: Generating 16GB of wal every second
(2^34 bytes /sec) is probably not impossible (cpu <> memory bandwidth
has been > 20GB/sec for a while); and that leaves you 2^29 seconds of
database runtime; or about 17 years. Seeing that a cluster can be
`pg_upgrade`d (which doesn't reset cluster LSN) since PG 9.0 from at
least version PG 8.4.0 (2009) (and through pg_migrator, from 8.3.0)),
we can assume that clusters hitting LSN=2^63 will be a reasonable
possibility within the next few years. As the lifespan of a PG release
is about 5 years, it doesn't seem impossible that there will be actual
clusters that are going to hit this naturally in the lifespan of PG15.
It is also possible that someone fat-fingers pg_resetwal; and creates
a cluster with LSN >= 2^63; resulting in negative values in the
s.param3 field. Not likely, but we can force such situations; and as
such we should handle that gracefully.
> 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> the reasoning for having this function and it's named as *checkpoint*
> when it doesn't do anything specific to the checkpoint at all?
I hadn't thought of using the types' inout functions, but it looks
like timestamp IO functions use a formatted timestring, which won't
work with the epoch-based timestamp stored in the view.
> Having 3 unnecessary functions that aren't useful to the users at all
> in proc.dat will simply eatup the function oids IMO. Hence, I suggest
> let's try to do without extra functions.
I agree that (1) could be simplified, or at least fully expressed in
SQL without exposing too many internals. If we're fine with exposing
internals like flags and type layouts, then (2), and arguably (4), can
be expressed in SQL as well.
-Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
@ 2022-03-01 08:57 ` Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-03-01 08:57 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > 3) Why do we need this extra calculation for start_lsn?
> > Do you ever see a negative LSN or something here?
> > + ('0/0'::pg_lsn + (
> > + CASE
> > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > + ELSE (0)::numeric
> > + END + (s.param3)::numeric)) AS start_lsn,
>
> Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> bigint type; thus the signed int64. This cast is OK as it wraps
> around, but that means we have to take care to correctly display the
> LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> the special-casing for negative values.
Yes. The extra calculation is required here as we are storing unit64
value in the variable of type int64. When we convert uint64 to int64
then the bit pattern is preserved (so no data is lost). The high-order
bit becomes the sign bit and if the sign bit is set, both the sign and
magnitude of the value changes. To safely get the actual uint64 value
whatever was assigned, we need the above calculations.
> > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > the reasoning for having this function and it's named as *checkpoint*
> > when it doesn't do anything specific to the checkpoint at all?
>
> I hadn't thought of using the types' inout functions, but it looks
> like timestamp IO functions use a formatted timestring, which won't
> work with the epoch-based timestamp stored in the view.
There is a variation of to_timestamp() which takes UNIX epoch (float8)
as an argument and converts it to timestamptz but we cannot directly
call this function with S.param4.
TimestampTz
GetCurrentTimestamp(void)
{
TimestampTz result;
struct timeval tp;
gettimeofday(&tp, NULL);
result = (TimestampTz) tp.tv_sec -
((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY);
result = (result * USECS_PER_SEC) + tp.tv_usec;
return result;
}
S.param4 contains the output of the above function
(GetCurrentTimestamp()) which returns Postgres epoch but the
to_timestamp() expects UNIX epoch as input. So some calculation is
required here. I feel the SQL 'to_timestamp(946684800 +
(S.param4::float / 1000000)) AS start_time' works fine. The value
'946684800' is equal to ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) *
SECS_PER_DAY). I am not sure whether it is good practice to use this
way. Kindly share your thoughts.
Thanks & Regards,
Nitin Jadhav
On Mon, Feb 28, 2022 at 6:40 PM Matthias van de Meent
<[email protected]> wrote:
>
> On Sun, 27 Feb 2022 at 16:14, Bharath Rupireddy
> <[email protected]> wrote:
> > 3) Why do we need this extra calculation for start_lsn?
> > Do you ever see a negative LSN or something here?
> > + ('0/0'::pg_lsn + (
> > + CASE
> > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > + ELSE (0)::numeric
> > + END + (s.param3)::numeric)) AS start_lsn,
>
> Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> bigint type; thus the signed int64. This cast is OK as it wraps
> around, but that means we have to take care to correctly display the
> LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> the special-casing for negative values.
>
> As to whether it is reasonable: Generating 16GB of wal every second
> (2^34 bytes /sec) is probably not impossible (cpu <> memory bandwidth
> has been > 20GB/sec for a while); and that leaves you 2^29 seconds of
> database runtime; or about 17 years. Seeing that a cluster can be
> `pg_upgrade`d (which doesn't reset cluster LSN) since PG 9.0 from at
> least version PG 8.4.0 (2009) (and through pg_migrator, from 8.3.0)),
> we can assume that clusters hitting LSN=2^63 will be a reasonable
> possibility within the next few years. As the lifespan of a PG release
> is about 5 years, it doesn't seem impossible that there will be actual
> clusters that are going to hit this naturally in the lifespan of PG15.
>
> It is also possible that someone fat-fingers pg_resetwal; and creates
> a cluster with LSN >= 2^63; resulting in negative values in the
> s.param3 field. Not likely, but we can force such situations; and as
> such we should handle that gracefully.
>
> > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > the reasoning for having this function and it's named as *checkpoint*
> > when it doesn't do anything specific to the checkpoint at all?
>
> I hadn't thought of using the types' inout functions, but it looks
> like timestamp IO functions use a formatted timestring, which won't
> work with the epoch-based timestamp stored in the view.
>
> > Having 3 unnecessary functions that aren't useful to the users at all
> > in proc.dat will simply eatup the function oids IMO. Hence, I suggest
> > let's try to do without extra functions.
>
> I agree that (1) could be simplified, or at least fully expressed in
> SQL without exposing too many internals. If we're fine with exposing
> internals like flags and type layouts, then (2), and arguably (4), can
> be expressed in SQL as well.
>
> -Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-02 11:15 ` Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-03 15:00 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-03-03 18:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
0 siblings, 3 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-03-02 11:15 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Thanks for reviewing.
> > > I suggested upthread to store the starting timeline instead. This way you can
> > > deduce whether it's a restartpoint or a checkpoint, but you can also deduce
> > > other information, like what was the starting WAL.
> >
> > I don't understand why we need the timeline here to just determine
> > whether it's a restartpoint or checkpoint.
>
> I'm not saying it's necessary, I'm saying that for the same space usage we can
> store something a bit more useful. If no one cares about having the starting
> timeline available for no extra cost then sure, let's just store the kind
> directly.
Fixed.
> 2) Can't we just have these checks inside CASE-WHEN-THEN-ELSE blocks
> directly instead of new function pg_stat_get_progress_checkpoint_kind?
> + snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
> + (flags == 0) ? "unknown" : "",
> + (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
> + (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
> + (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
> + (flags & CHECKPOINT_FORCE) ? "force " : "",
> + (flags & CHECKPOINT_WAIT) ? "wait " : "",
> + (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
> + (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
> + (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
Fixed.
---
> 5) Do we need a special phase for this checkpoint operation? I'm not
> sure in which cases it will take a long time, but it looks like
> there's a wait loop here.
> vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
> if (nvxids > 0)
> {
> do
> {
> pg_usleep(10000L); /* wait for 10 msec */
> } while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids));
> }
Yes. It is better to add a separate phase here.
---
> Also, how about special phases for SyncPostCheckpoint(),
> SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
> PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
> it might be increase in future (?)), TruncateSUBTRANS()?
SyncPreCheckpoint() is just incrementing a counter and
PreallocXlogFiles() currently pre-allocates only 1 WAL file. I feel
there is no need to add any phases for these as of now. We can add in
the future if necessary. Added phases for SyncPostCheckpoint(),
InvalidateObsoleteReplicationSlots() and TruncateSUBTRANS().
---
> 6) SLRU (Simple LRU) isn't a phase here, you can just say
> PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES.
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
> CheckPointPredicate();
>
> And :s/checkpointing SLRU pages/checkpointing predicate lock pages
>+ WHEN 9 THEN 'checkpointing SLRU pages'
Fixed.
---
> 7) :s/PROGRESS_CHECKPOINT_PHASE_FILE_SYNC/PROGRESS_CHECKPOINT_PHASE_PROCESS_FILE_SYNC_REQUESTS
I feel PROGRESS_CHECKPOINT_PHASE_FILE_SYNC is a better option here as
it describes the purpose in less words.
> And :s/WHEN 11 THEN 'performing sync requests'/WHEN 11 THEN
> 'processing file sync requests'
Fixed.
---
> 8) :s/Finalizing/finalizing
> + WHEN 14 THEN 'Finalizing'
Fixed.
---
> 9) :s/checkpointing snapshots/checkpointing logical replication snapshot files
> + WHEN 3 THEN 'checkpointing snapshots'
> :s/checkpointing logical rewrite mappings/checkpointing logical
> replication rewrite mapping files
> + WHEN 4 THEN 'checkpointing logical rewrite mappings'
Fixed.
---
> 10) I'm not sure if it's discussed, how about adding the number of
> snapshot/mapping files so far the checkpoint has processed in file
> processing while loops of
> CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
> be many logical snapshot or mapping files and users may be interested
> in knowing the so-far-processed-file-count.
I had thought about this while sharing the v1 patch and mentioned my
views upthread. I feel it won't give meaningful progress information
(It can be treated as statistics). Hence not included. Thoughts?
> > > As mentioned upthread, there can be multiple backends that request a
> > > checkpoint, so unless we want to store an array of pid we should store a number
> > > of backend that are waiting for a new checkpoint.
> >
> > Yeah, you are right. Let's not go that path and store an array of
> > pids. I don't see a strong use-case with the pid of the process
> > requesting checkpoint. If required, we can add it later once the
> > pg_stat_progress_checkpoint view gets in.
>
> I don't think that's really necessary to give the pid list.
>
> If you requested a new checkpoint, it doesn't matter if it's only your backend
> that triggered it, another backend or a few other dozen, the result will be the
> same and you have the information that the request has been seen. We could
> store just a bool for that but having a number instead also gives a bit more
> information and may allow you to detect some broken logic on your client code
> if it keeps increasing.
It's a good metric to show in the view but the information is not
readily available. Additional code is required to calculate the number
of requests. Is it worth doing that? I feel this can be added later if
required.
Please find the v4 patch attached and share your thoughts.
Thanks & Regards,
Nitin Jadhav
On Tue, Mar 1, 2022 at 2:27 PM Nitin Jadhav
<[email protected]> wrote:
>
> > > 3) Why do we need this extra calculation for start_lsn?
> > > Do you ever see a negative LSN or something here?
> > > + ('0/0'::pg_lsn + (
> > > + CASE
> > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > + ELSE (0)::numeric
> > > + END + (s.param3)::numeric)) AS start_lsn,
> >
> > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > bigint type; thus the signed int64. This cast is OK as it wraps
> > around, but that means we have to take care to correctly display the
> > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > the special-casing for negative values.
>
> Yes. The extra calculation is required here as we are storing unit64
> value in the variable of type int64. When we convert uint64 to int64
> then the bit pattern is preserved (so no data is lost). The high-order
> bit becomes the sign bit and if the sign bit is set, both the sign and
> magnitude of the value changes. To safely get the actual uint64 value
> whatever was assigned, we need the above calculations.
>
> > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > the reasoning for having this function and it's named as *checkpoint*
> > > when it doesn't do anything specific to the checkpoint at all?
> >
> > I hadn't thought of using the types' inout functions, but it looks
> > like timestamp IO functions use a formatted timestring, which won't
> > work with the epoch-based timestamp stored in the view.
>
> There is a variation of to_timestamp() which takes UNIX epoch (float8)
> as an argument and converts it to timestamptz but we cannot directly
> call this function with S.param4.
>
> TimestampTz
> GetCurrentTimestamp(void)
> {
> TimestampTz result;
> struct timeval tp;
>
> gettimeofday(&tp, NULL);
>
> result = (TimestampTz) tp.tv_sec -
> ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY);
> result = (result * USECS_PER_SEC) + tp.tv_usec;
>
> return result;
> }
>
> S.param4 contains the output of the above function
> (GetCurrentTimestamp()) which returns Postgres epoch but the
> to_timestamp() expects UNIX epoch as input. So some calculation is
> required here. I feel the SQL 'to_timestamp(946684800 +
> (S.param4::float / 1000000)) AS start_time' works fine. The value
> '946684800' is equal to ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) *
> SECS_PER_DAY). I am not sure whether it is good practice to use this
> way. Kindly share your thoughts.
>
> Thanks & Regards,
> Nitin Jadhav
>
> On Mon, Feb 28, 2022 at 6:40 PM Matthias van de Meent
> <[email protected]> wrote:
> >
> > On Sun, 27 Feb 2022 at 16:14, Bharath Rupireddy
> > <[email protected]> wrote:
> > > 3) Why do we need this extra calculation for start_lsn?
> > > Do you ever see a negative LSN or something here?
> > > + ('0/0'::pg_lsn + (
> > > + CASE
> > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > + ELSE (0)::numeric
> > > + END + (s.param3)::numeric)) AS start_lsn,
> >
> > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > bigint type; thus the signed int64. This cast is OK as it wraps
> > around, but that means we have to take care to correctly display the
> > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > the special-casing for negative values.
> >
> > As to whether it is reasonable: Generating 16GB of wal every second
> > (2^34 bytes /sec) is probably not impossible (cpu <> memory bandwidth
> > has been > 20GB/sec for a while); and that leaves you 2^29 seconds of
> > database runtime; or about 17 years. Seeing that a cluster can be
> > `pg_upgrade`d (which doesn't reset cluster LSN) since PG 9.0 from at
> > least version PG 8.4.0 (2009) (and through pg_migrator, from 8.3.0)),
> > we can assume that clusters hitting LSN=2^63 will be a reasonable
> > possibility within the next few years. As the lifespan of a PG release
> > is about 5 years, it doesn't seem impossible that there will be actual
> > clusters that are going to hit this naturally in the lifespan of PG15.
> >
> > It is also possible that someone fat-fingers pg_resetwal; and creates
> > a cluster with LSN >= 2^63; resulting in negative values in the
> > s.param3 field. Not likely, but we can force such situations; and as
> > such we should handle that gracefully.
> >
> > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > the reasoning for having this function and it's named as *checkpoint*
> > > when it doesn't do anything specific to the checkpoint at all?
> >
> > I hadn't thought of using the types' inout functions, but it looks
> > like timestamp IO functions use a formatted timestring, which won't
> > work with the epoch-based timestamp stored in the view.
> >
> > > Having 3 unnecessary functions that aren't useful to the users at all
> > > in proc.dat will simply eatup the function oids IMO. Hence, I suggest
> > > let's try to do without extra functions.
> >
> > I agree that (1) could be simplified, or at least fully expressed in
> > SQL without exposing too many internals. If we're fine with exposing
> > internals like flags and type layouts, then (2), and arguably (4), can
> > be expressed in SQL as well.
> >
> > -Matthias
Attachments:
[application/octet-stream] v4-0001-pg_stat_progress_checkpoint-view.patch (34.9K, ../../CAMm1aWYWmqys2O7UekQ0=ZSWSpG9h2qoEW7w46yiWM0eO1Lu7g@mail.gmail.com/2-v4-0001-pg_stat_progress_checkpoint-view.patch)
download | inline diff:
From a3ff5c7b194d1128ab0272e546913baeafc04a0c Mon Sep 17 00:00:00 2001
From: Nitin Jadhav <[email protected]>
Date: Wed, 2 Mar 2022 11:06:57 +0000
Subject: [PATCH] pg_stat_progress_checkpoint view
---
doc/src/sgml/monitoring.sgml | 384 +++++++++++++++++++++++++++
doc/src/sgml/ref/checkpoint.sgml | 6 +
doc/src/sgml/wal.sgml | 5 +-
src/backend/access/transam/xlog.c | 98 +++++++
src/backend/catalog/system_views.sql | 50 ++++
src/backend/storage/buffer/bufmgr.c | 7 +
src/backend/storage/sync/sync.c | 6 +
src/backend/utils/adt/pgstatfuncs.c | 2 +
src/include/commands/progress.h | 36 +++
src/include/utils/backend_progress.h | 3 +-
src/test/regress/expected/rules.out | 76 ++++++
11 files changed, 671 insertions(+), 2 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 9fb62fec8e..8d7c8ffc92 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -401,6 +401,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the checkpoint.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -6844,6 +6851,383 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>Checkpoint Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the checkpointer process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>kind</structfield> <type>text</type>
+ </para>
+ <para>
+ Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_lsn</structfield> <type>text</type>
+ </para>
+ <para>
+ The checkpoint start location.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_time</structfield> <type>timestamp with time zone</type>
+ </para>
+ <para>
+ Elapsed time of the checkpoint.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal>buffers_total</literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>buffers_total</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>files_total</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-types">
+ <title>Checkpoint types</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Types</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>checkpoint</literal></entry>
+ <entry>
+ The current operation is checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>restartpoint</literal></entry>
+ <entry>
+ The current operation is restartpoint.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-kinds">
+ <title>Checkpoint kinds</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Kinds</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>shutdown</literal></entry>
+ <entry>
+ The checkpoint is for shutdown.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>end-of-recovery</literal></entry>
+ <entry>
+ The checkpoint is for end-of-recovery.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>immediate</literal></entry>
+ <entry>
+ The checkpoint is happens without delays.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint is started because some operation forced a checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>flush all</literal></entry>
+ <entry>
+ The checkpoint flushes all pages, including those belonging to unlogged
+ tables.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wait</literal></entry>
+ <entry>
+ Wait for completion before returning.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>requested</literal></entry>
+ <entry>
+ The checkpoint request has been made.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>max_wal_size</literal> is
+ reached.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>checkpoint_timeout</literal>
+ expired.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>Checkpoint phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The checkpointer process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>getting virtual transaction IDs</literal></entry>
+ <entry>
+ The checkpointer process is getting the virtual transaction IDs that
+ are delaying the checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently flushing all the replication slots
+ to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical replication snapshot files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing all the serialized
+ snapshot files that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mapping files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing/flushing the logical
+ rewrite mapping files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit log pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit log pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit time stamp pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit time stamp pages to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing subtransaction pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing subtransaction pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing multixact pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing multixact pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing predicate lock pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing predicate lock pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The checkpointer process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>processing file sync requests</literal></entry>
+ <entry>
+ The checkpointer process is currently processing file sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The checkpointer process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing post checkpoint cleanup</literal></entry>
+ <entry>
+ The checkpointer process is currently performing post checkpoint cleanup.
+ It removes any lingering files that can be safely removed.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>invalidating replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently invalidating replication slots.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old XLOG files</literal></entry>
+ <entry>
+ The checkpointer process is currently recycling old XLOG files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>truncating subtransactions</literal></entry>
+ <entry>
+ The checkpointer process is currently removing the subtransaction
+ segments.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The checkpointer process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/doc/src/sgml/ref/checkpoint.sgml b/doc/src/sgml/ref/checkpoint.sgml
index 1cebc03d15..a88c76533a 100644
--- a/doc/src/sgml/ref/checkpoint.sgml
+++ b/doc/src/sgml/ref/checkpoint.sgml
@@ -56,6 +56,12 @@ CHECKPOINT
the <link linkend="predefined-roles-table"><literal>pg_checkpointer</literal></link>
role can call <command>CHECKPOINT</command>.
</para>
+
+ <para>
+ The checkpointer process running the checkpoint will report its progress
+ in the <structname>pg_stat_progress_checkpoint</structname> view. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
+ </para>
</refsect1>
<refsect1>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 2bb27a8468..a75d1d63d0 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -530,7 +530,10 @@
adjust the <xref linkend="guc-archive-timeout"/> parameter rather than the
checkpoint parameters.)
It is also possible to force a checkpoint by using the SQL
- command <command>CHECKPOINT</command>.
+ command <command>CHECKPOINT</command>. The checkpointer process running the
+ checkpoint will report its progress in the
+ <structname>pg_stat_progress_checkpoint</structname> view. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
</para>
<para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 0d2bd7a357..9591f2cecc 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -65,6 +65,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "commands/progress.h"
#include "common/controldata_utils.h"
#include "executor/instrument.h"
#include "miscadmin.h"
@@ -719,6 +720,8 @@ static void WALInsertLockAcquireExclusive(void);
static void WALInsertLockRelease(void);
static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static void checkpoint_progress_start(int flags, int type);
+
/*
* Insert an XLOG record represented by an already-constructed chain of data
* chunks. This is a low-level routine; to construct the WAL record header
@@ -6296,6 +6299,9 @@ CreateCheckPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_CHECKPOINT);
+
/*
* Use a critical section to force system panic if we have trouble.
*/
@@ -6394,6 +6400,7 @@ CreateCheckPoint(int flags)
curInsert += SizeOfXLogShortPHD;
}
checkPoint.redo = curInsert;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
/*
* Here we update the shared RedoRecPtr for future XLogInsert calls; this
@@ -6501,6 +6508,8 @@ CreateCheckPoint(int flags)
* and we will correctly flush the update below. So we cannot miss any
* xacts we need to wait for.
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS);
vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
if (nvxids > 0)
{
@@ -6604,6 +6613,8 @@ CreateCheckPoint(int flags)
/*
* Let smgr do post-checkpoint cleanup (eg, deleting old files).
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP);
SyncPostCheckpoint();
/*
@@ -6619,6 +6630,9 @@ CreateCheckPoint(int flags)
*/
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
KeepLogSeg(recptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -6629,6 +6643,8 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
@@ -6647,11 +6663,21 @@ CreateCheckPoint(int flags)
* StartupSUBTRANS hasn't been called yet.
*/
if (!RecoveryInProgress())
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -6808,29 +6834,60 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES);
CheckPointPredicate();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FILE_SYNC);
ProcessSyncRequests();
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -6977,6 +7034,9 @@ CreateRestartPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the restartpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT);
+
if (log_checkpoints)
LogCheckpointStart(flags, true);
@@ -7051,6 +7111,9 @@ CreateRestartPoint(int flags)
replayPtr = GetXLogReplayRecPtr(&replayTLI);
endptr = (receivePtr < replayPtr) ? replayPtr : receivePtr;
KeepLogSeg(endptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -7077,6 +7140,8 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
/*
@@ -7093,11 +7158,20 @@ CreateRestartPoint(int flags)
* this because StartupSUBTRANS hasn't been called yet.
*/
if (EnableHotStandby)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the restartpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -9197,3 +9271,27 @@ SetWalWriterSleeping(bool sleeping)
XLogCtl->WalWriterSleeping = sleeping;
SpinLockRelease(&XLogCtl->info_lck);
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+static void
+checkpoint_progress_start(int flags, int type)
+{
+ const int index[] = {
+ PROGRESS_CHECKPOINT_TYPE,
+ PROGRESS_CHECKPOINT_KIND,
+ PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_START_TIMESTAMP
+ };
+ int64 val[4];
+
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+
+ val[0] = type;
+ val[1] = flags;
+ val[2] = PROGRESS_CHECKPOINT_PHASE_INIT;
+ val[3] = CheckpointStats.ckpt_start_t;
+
+ pgstat_progress_update_multi_param(4, index, val);
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 40b7bca5a9..1e5b6e995f 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1273,3 +1273,53 @@ CREATE VIEW pg_stat_subscription_stats AS
ss.stats_reset
FROM pg_subscription as s,
pg_stat_get_subscription_stats(s.oid) as ss;
+
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ CASE S.param1 WHEN 1 THEN 'checkpoint'
+ WHEN 2 THEN 'restartpoint'
+ END AS type,
+ ( CASE WHEN (S.param2 & 1) > 0 THEN 'shutdown ' ELSE '' END ||
+ CASE WHEN (S.param2 & 2) > 0 THEN 'end-of-recovery ' ELSE '' END ||
+ CASE WHEN (S.param2 & 4) > 0 THEN 'immediate ' ELSE '' END ||
+ CASE WHEN (S.param2 & 8) > 0 THEN 'force ' ELSE '' END ||
+ CASE WHEN (S.param2 & 16) > 0 THEN 'flush-all ' ELSE '' END ||
+ CASE WHEN (S.param2 & 32) > 0 THEN 'wait ' ELSE '' END ||
+ CASE WHEN (S.param2 & 64) > 0 THEN 'requested ' ELSE '' END ||
+ CASE WHEN (S.param2 & 128) > 0 THEN 'wal ' ELSE '' END ||
+ CASE WHEN (S.param2 & 256) > 0 THEN 'time ' ELSE '' END
+ ) AS kind,
+ ( '0/0'::pg_lsn +
+ ((CASE
+ WHEN S.param3 < 0 THEN pow(2::numeric, 64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ S.param3::numeric)
+ ) AS start_lsn,
+ to_timestamp(946684800 + (S.param4::float8 / 1000000)) AS start_time,
+ CASE S.param5 WHEN 1 THEN 'initializing'
+ WHEN 2 THEN 'getting virtual transaction IDs'
+ WHEN 3 THEN 'checkpointing replication slots'
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'
+ WHEN 6 THEN 'checkpointing commit log pages'
+ WHEN 7 THEN 'checkpointing commit time stamp pages'
+ WHEN 8 THEN 'checkpointing subtransaction pages'
+ WHEN 9 THEN 'checkpointing multixact pages'
+ WHEN 10 THEN 'checkpointing predicate lock pages'
+ WHEN 11 THEN 'checkpointing buffers'
+ WHEN 12 THEN 'processing file sync requests'
+ WHEN 13 THEN 'performing two phase checkpoint'
+ WHEN 14 THEN 'performing post checkpoint cleanup'
+ WHEN 15 THEN 'invalidating replication slots'
+ WHEN 16 THEN 'recycling old XLOG files'
+ WHEN 17 THEN 'truncating subtransactions'
+ WHEN 18 THEN 'finalizing'
+ END AS phase,
+ S.param6 AS buffers_total,
+ S.param7 AS buffers_processed,
+ S.param8 AS buffers_written,
+ S.param9 AS files_total,
+ S.param10 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index f5459c68f8..9663035d7a 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -38,6 +38,7 @@
#include "access/xlogutils.h"
#include "catalog/catalog.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2012,6 +2013,8 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_TOTAL,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2129,6 +2132,8 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2149,6 +2154,8 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.m_buf_written_checkpoints++;
num_written++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index e161d57761..638d3eb781 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -356,6 +357,9 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_TOTAL,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -419,6 +423,8 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index fd993d0d5f..95df730415 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -494,6 +494,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..7064026bf1 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,40 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_TYPE 0
+#define PROGRESS_CHECKPOINT_KIND 1
+#define PROGRESS_CHECKPOINT_LSN 2
+#define PROGRESS_CHECKPOINT_START_TIMESTAMP 3
+#define PROGRESS_CHECKPOINT_PHASE 4
+#define PROGRESS_CHECKPOINT_BUFFERS_TOTAL 5
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 6
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 7
+#define PROGRESS_CHECKPOINT_FILES_TOTAL 8
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 9
+
+/* Types of checkpoint (as advertised via PROGRESS_CHECKPOINT_TYPE) */
+#define PROGRESS_CHECKPOINT_TYPE_CHECKPOINT 1
+#define PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT 2
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 1
+#define PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS 2
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 3
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 4
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 5
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 6
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 9
+#define PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES 10
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 11
+#define PROGRESS_CHECKPOINT_PHASE_FILE_SYNC 12
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 13
+#define PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP 14
+#define PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS 15
+#define PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE 16
+#define PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS 17
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 18
+
#endif
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index ac468568a1..11626a4200 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1897,6 +1897,82 @@ pg_stat_progress_basebackup| SELECT s.pid,
s.param4 AS tablespaces_total,
s.param5 AS tablespaces_streamed
FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
+pg_stat_progress_checkpoint| SELECT s.pid,
+ CASE s.param1
+ WHEN 1 THEN 'checkpoint'::text
+ WHEN 2 THEN 'restartpoint'::text
+ ELSE NULL::text
+ END AS type,
+ ((((((((
+ CASE
+ WHEN ((s.param2 & (1)::bigint) > 0) THEN 'shutdown '::text
+ ELSE ''::text
+ END ||
+ CASE
+ WHEN ((s.param2 & (2)::bigint) > 0) THEN 'end-of-recovery '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (4)::bigint) > 0) THEN 'immediate '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (8)::bigint) > 0) THEN 'force '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (16)::bigint) > 0) THEN 'flush-all '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (32)::bigint) > 0) THEN 'wait '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (64)::bigint) > 0) THEN 'requested '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (128)::bigint) > 0) THEN 'wal '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (256)::bigint) > 0) THEN 'time '::text
+ ELSE ''::text
+ END) AS kind,
+ ('0/0'::pg_lsn + (
+ CASE
+ WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (s.param3)::numeric)) AS start_lsn,
+ to_timestamp(((946684800)::double precision + ((s.param4)::double precision / (1000000)::double precision))) AS start_time,
+ CASE s.param5
+ WHEN 1 THEN 'initializing'::text
+ WHEN 2 THEN 'getting virtual transaction IDs'::text
+ WHEN 3 THEN 'checkpointing replication slots'::text
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'::text
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'::text
+ WHEN 6 THEN 'checkpointing commit log pages'::text
+ WHEN 7 THEN 'checkpointing commit time stamp pages'::text
+ WHEN 8 THEN 'checkpointing subtransaction pages'::text
+ WHEN 9 THEN 'checkpointing multixact pages'::text
+ WHEN 10 THEN 'checkpointing predicate lock pages'::text
+ WHEN 11 THEN 'checkpointing buffers'::text
+ WHEN 12 THEN 'processing file sync requests'::text
+ WHEN 13 THEN 'performing two phase checkpoint'::text
+ WHEN 14 THEN 'performing post checkpoint cleanup'::text
+ WHEN 15 THEN 'invalidating replication slots'::text
+ WHEN 16 THEN 'recycling old XLOG files'::text
+ WHEN 17 THEN 'truncating subtransactions'::text
+ WHEN 18 THEN 'finalizing'::text
+ ELSE NULL::text
+ END AS phase,
+ s.param6 AS buffers_total,
+ s.param7 AS buffers_processed,
+ s.param8 AS buffers_written,
+ s.param9 AS files_total,
+ s.param10 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
pg_stat_progress_cluster| SELECT s.pid,
s.datid,
d.datname,
--
2.25.1
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-02 18:22 ` Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2 siblings, 1 reply; 85+ messages in thread
From: Bharath Rupireddy @ 2022-03-02 18:22 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Mar 2, 2022 at 4:45 PM Nitin Jadhav
<[email protected]> wrote:
> > Also, how about special phases for SyncPostCheckpoint(),
> > SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
> > PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
> > it might be increase in future (?)), TruncateSUBTRANS()?
>
> SyncPreCheckpoint() is just incrementing a counter and
> PreallocXlogFiles() currently pre-allocates only 1 WAL file. I feel
> there is no need to add any phases for these as of now. We can add in
> the future if necessary. Added phases for SyncPostCheckpoint(),
> InvalidateObsoleteReplicationSlots() and TruncateSUBTRANS().
Okay.
> > 10) I'm not sure if it's discussed, how about adding the number of
> > snapshot/mapping files so far the checkpoint has processed in file
> > processing while loops of
> > CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
> > be many logical snapshot or mapping files and users may be interested
> > in knowing the so-far-processed-file-count.
>
> I had thought about this while sharing the v1 patch and mentioned my
> views upthread. I feel it won't give meaningful progress information
> (It can be treated as statistics). Hence not included. Thoughts?
Okay. If there are any complaints about it we can always add them later.
> > > > As mentioned upthread, there can be multiple backends that request a
> > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > of backend that are waiting for a new checkpoint.
> > >
> > > Yeah, you are right. Let's not go that path and store an array of
> > > pids. I don't see a strong use-case with the pid of the process
> > > requesting checkpoint. If required, we can add it later once the
> > > pg_stat_progress_checkpoint view gets in.
> >
> > I don't think that's really necessary to give the pid list.
> >
> > If you requested a new checkpoint, it doesn't matter if it's only your backend
> > that triggered it, another backend or a few other dozen, the result will be the
> > same and you have the information that the request has been seen. We could
> > store just a bool for that but having a number instead also gives a bit more
> > information and may allow you to detect some broken logic on your client code
> > if it keeps increasing.
>
> It's a good metric to show in the view but the information is not
> readily available. Additional code is required to calculate the number
> of requests. Is it worth doing that? I feel this can be added later if
> required.
Yes, we can always add it later if required.
> Please find the v4 patch attached and share your thoughts.
I reviewed v4 patch, here are my comments:
1) Can we convert below into pgstat_progress_update_multi_param, just
to avoid function calls?
pgstat_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
2) Why are we not having special phase for CheckPointReplicationOrigin
as it does good bunch of work (writing to disk, XLogFlush,
durable_rename) especially when max_replication_slots is large?
3) I don't think "requested" is necessary here as it doesn't add any
value or it's not a checkpoint kind or such, you can remove it.
4) s:/'recycling old XLOG files'/'recycling old WAL files'
+ WHEN 16 THEN 'recycling old XLOG files'
5) Can we place CREATE VIEW pg_stat_progress_checkpoint AS definition
next to pg_stat_progress_copy in system_view.sql? It looks like all
the progress reporting views are next to each other.
6) How about shutdown and end-of-recovery checkpoint? Are you planning
to have an ereport_startup_progress mechanism as 0002?
7) I think you don't need to call checkpoint_progress_start and
pgstat_progress_update_param, any other progress reporting function
for shutdown and end-of-recovery checkpoint right?
8) Not for all kinds of checkpoints right? pg_stat_progress_checkpoint
can't show progress report for shutdown and end-of-recovery
checkpoint, I think you need to specify that here in wal.sgml and
checkpoint.sgml.
+ command <command>CHECKPOINT</command>. The checkpointer process running the
+ checkpoint will report its progress in the
+ <structname>pg_stat_progress_checkpoint</structname> view. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
9) Can you add a test case for pg_stat_progress_checkpoint view? I
think it's good to add one. See, below for reference:
-- Add a trigger to catch and print the contents of the catalog view
-- pg_stat_progress_copy during data insertion. This allows to test
-- the validation of some progress reports for COPY FROM where the trigger
-- would fire.
create function notice_after_tab_progress_reporting() returns trigger AS
$$
declare report record;
10) Typo: it's not "is happens"
+ The checkpoint is happens without delays.
11) Can you be specific what are those "some operations" that forced a
checkpoint? May be like, basebackup, createdb or something?
+ The checkpoint is started because some operation forced a checkpoint.
12) Can you be a bit elobartive here who waits? Something like the
backend that requested checkpoint will wait until it's completion ....
+ Wait for completion before returning.
13) "removing unneeded or flushing needed logical rewrite mapping files"
+ The checkpointer process is currently removing/flushing the logical
14) "old WAL files"
+ The checkpointer process is currently recycling old XLOG files.
Regards,
Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
@ 2022-03-07 14:15 ` Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-03-07 14:15 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> 1) Can we convert below into pgstat_progress_update_multi_param, just
> to avoid function calls?
> pgstat_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
> pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
>
> 2) Why are we not having special phase for CheckPointReplicationOrigin
> as it does good bunch of work (writing to disk, XLogFlush,
> durable_rename) especially when max_replication_slots is large?
>
> 3) I don't think "requested" is necessary here as it doesn't add any
> value or it's not a checkpoint kind or such, you can remove it.
>
> 4) s:/'recycling old XLOG files'/'recycling old WAL files'
> + WHEN 16 THEN 'recycling old XLOG files'
>
> 5) Can we place CREATE VIEW pg_stat_progress_checkpoint AS definition
> next to pg_stat_progress_copy in system_view.sql? It looks like all
> the progress reporting views are next to each other.
I will take care in the next patch.
---
> 6) How about shutdown and end-of-recovery checkpoint? Are you planning
> to have an ereport_startup_progress mechanism as 0002?
I thought of including it earlier then I felt lets first make the
current patch stable. Once all the fields are properly decided and the
patch gets in then we can easily extend the functionality to shutdown
and end-of-recovery cases. I have also observed that the timer
functionality wont work properly in case of shutdown as we are doing
an immediate checkpoint. So this needs a lot of discussion and I would
like to handle this on a separate thread.
---
> 7) I think you don't need to call checkpoint_progress_start and
> pgstat_progress_update_param, any other progress reporting function
> for shutdown and end-of-recovery checkpoint right?
I had included the guards earlier and then removed later based on the
discussion upthread.
---
> 8) Not for all kinds of checkpoints right? pg_stat_progress_checkpoint
> can't show progress report for shutdown and end-of-recovery
> checkpoint, I think you need to specify that here in wal.sgml and
> checkpoint.sgml.
> + command <command>CHECKPOINT</command>. The checkpointer process running the
> + checkpoint will report its progress in the
> + <structname>pg_stat_progress_checkpoint</structname> view. See
> + <xref linkend="checkpoint-progress-reporting"/> for details.
>
> 9) Can you add a test case for pg_stat_progress_checkpoint view? I
> think it's good to add one. See, below for reference:
> -- Add a trigger to catch and print the contents of the catalog view
> -- pg_stat_progress_copy during data insertion. This allows to test
> -- the validation of some progress reports for COPY FROM where the trigger
> -- would fire.
> create function notice_after_tab_progress_reporting() returns trigger AS
> $$
> declare report record;
>
> 10) Typo: it's not "is happens"
> + The checkpoint is happens without delays.
>
> 11) Can you be specific what are those "some operations" that forced a
> checkpoint? May be like, basebackup, createdb or something?
> + The checkpoint is started because some operation forced a checkpoint.
>
> 12) Can you be a bit elobartive here who waits? Something like the
> backend that requested checkpoint will wait until it's completion ....
> + Wait for completion before returning.
>
> 13) "removing unneeded or flushing needed logical rewrite mapping files"
> + The checkpointer process is currently removing/flushing the logical
>
> 14) "old WAL files"
> + The checkpointer process is currently recycling old XLOG files.
I will take care in the next patch.
Thanks & Regards,
Nitin Jadhav
On Wed, Mar 2, 2022 at 11:52 PM Bharath Rupireddy
<[email protected]> wrote:
>
> On Wed, Mar 2, 2022 at 4:45 PM Nitin Jadhav
> <[email protected]> wrote:
> > > Also, how about special phases for SyncPostCheckpoint(),
> > > SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
> > > PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
> > > it might be increase in future (?)), TruncateSUBTRANS()?
> >
> > SyncPreCheckpoint() is just incrementing a counter and
> > PreallocXlogFiles() currently pre-allocates only 1 WAL file. I feel
> > there is no need to add any phases for these as of now. We can add in
> > the future if necessary. Added phases for SyncPostCheckpoint(),
> > InvalidateObsoleteReplicationSlots() and TruncateSUBTRANS().
>
> Okay.
>
> > > 10) I'm not sure if it's discussed, how about adding the number of
> > > snapshot/mapping files so far the checkpoint has processed in file
> > > processing while loops of
> > > CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
> > > be many logical snapshot or mapping files and users may be interested
> > > in knowing the so-far-processed-file-count.
> >
> > I had thought about this while sharing the v1 patch and mentioned my
> > views upthread. I feel it won't give meaningful progress information
> > (It can be treated as statistics). Hence not included. Thoughts?
>
> Okay. If there are any complaints about it we can always add them later.
>
> > > > > As mentioned upthread, there can be multiple backends that request a
> > > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > > of backend that are waiting for a new checkpoint.
> > > >
> > > > Yeah, you are right. Let's not go that path and store an array of
> > > > pids. I don't see a strong use-case with the pid of the process
> > > > requesting checkpoint. If required, we can add it later once the
> > > > pg_stat_progress_checkpoint view gets in.
> > >
> > > I don't think that's really necessary to give the pid list.
> > >
> > > If you requested a new checkpoint, it doesn't matter if it's only your backend
> > > that triggered it, another backend or a few other dozen, the result will be the
> > > same and you have the information that the request has been seen. We could
> > > store just a bool for that but having a number instead also gives a bit more
> > > information and may allow you to detect some broken logic on your client code
> > > if it keeps increasing.
> >
> > It's a good metric to show in the view but the information is not
> > readily available. Additional code is required to calculate the number
> > of requests. Is it worth doing that? I feel this can be added later if
> > required.
>
> Yes, we can always add it later if required.
>
> > Please find the v4 patch attached and share your thoughts.
>
> I reviewed v4 patch, here are my comments:
>
> 1) Can we convert below into pgstat_progress_update_multi_param, just
> to avoid function calls?
> pgstat_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
> pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
>
> 2) Why are we not having special phase for CheckPointReplicationOrigin
> as it does good bunch of work (writing to disk, XLogFlush,
> durable_rename) especially when max_replication_slots is large?
>
> 3) I don't think "requested" is necessary here as it doesn't add any
> value or it's not a checkpoint kind or such, you can remove it.
>
> 4) s:/'recycling old XLOG files'/'recycling old WAL files'
> + WHEN 16 THEN 'recycling old XLOG files'
>
> 5) Can we place CREATE VIEW pg_stat_progress_checkpoint AS definition
> next to pg_stat_progress_copy in system_view.sql? It looks like all
> the progress reporting views are next to each other.
>
> 6) How about shutdown and end-of-recovery checkpoint? Are you planning
> to have an ereport_startup_progress mechanism as 0002?
>
> 7) I think you don't need to call checkpoint_progress_start and
> pgstat_progress_update_param, any other progress reporting function
> for shutdown and end-of-recovery checkpoint right?
>
> 8) Not for all kinds of checkpoints right? pg_stat_progress_checkpoint
> can't show progress report for shutdown and end-of-recovery
> checkpoint, I think you need to specify that here in wal.sgml and
> checkpoint.sgml.
> + command <command>CHECKPOINT</command>. The checkpointer process running the
> + checkpoint will report its progress in the
> + <structname>pg_stat_progress_checkpoint</structname> view. See
> + <xref linkend="checkpoint-progress-reporting"/> for details.
>
> 9) Can you add a test case for pg_stat_progress_checkpoint view? I
> think it's good to add one. See, below for reference:
> -- Add a trigger to catch and print the contents of the catalog view
> -- pg_stat_progress_copy during data insertion. This allows to test
> -- the validation of some progress reports for COPY FROM where the trigger
> -- would fire.
> create function notice_after_tab_progress_reporting() returns trigger AS
> $$
> declare report record;
>
> 10) Typo: it's not "is happens"
> + The checkpoint is happens without delays.
>
> 11) Can you be specific what are those "some operations" that forced a
> checkpoint? May be like, basebackup, createdb or something?
> + The checkpoint is started because some operation forced a checkpoint.
>
> 12) Can you be a bit elobartive here who waits? Something like the
> backend that requested checkpoint will wait until it's completion ....
> + Wait for completion before returning.
>
> 13) "removing unneeded or flushing needed logical rewrite mapping files"
> + The checkpointer process is currently removing/flushing the logical
>
> 14) "old WAL files"
> + The checkpointer process is currently recycling old XLOG files.
>
> Regards,
> Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-08 14:55 ` Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-03-08 14:55 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > 11) Can you be specific what are those "some operations" that forced a
> > checkpoint? May be like, basebackup, createdb or something?
> > + The checkpoint is started because some operation forced a checkpoint.
> >
> I will take care in the next patch.
I feel mentioning/listing the specific operation makes it difficult to
maintain the document. If we add any new functionality in future which
needs a force checkpoint, then there is a high chance that we will
miss to update here. Hence modified it to "The checkpoint is started
because some operation (for which the checkpoint is necessary) is
forced the checkpoint".
Fixed other comments as per the discussion above.
Please find the v5 patch attached and share your thoughts.
Thanks & Regards,
Nitin Jadhav
On Mon, Mar 7, 2022 at 7:45 PM Nitin Jadhav
<[email protected]> wrote:
>
> > 1) Can we convert below into pgstat_progress_update_multi_param, just
> > to avoid function calls?
> > pgstat_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
> > pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> >
> > 2) Why are we not having special phase for CheckPointReplicationOrigin
> > as it does good bunch of work (writing to disk, XLogFlush,
> > durable_rename) especially when max_replication_slots is large?
> >
> > 3) I don't think "requested" is necessary here as it doesn't add any
> > value or it's not a checkpoint kind or such, you can remove it.
> >
> > 4) s:/'recycling old XLOG files'/'recycling old WAL files'
> > + WHEN 16 THEN 'recycling old XLOG files'
> >
> > 5) Can we place CREATE VIEW pg_stat_progress_checkpoint AS definition
> > next to pg_stat_progress_copy in system_view.sql? It looks like all
> > the progress reporting views are next to each other.
>
> I will take care in the next patch.
> ---
>
> > 6) How about shutdown and end-of-recovery checkpoint? Are you planning
> > to have an ereport_startup_progress mechanism as 0002?
>
> I thought of including it earlier then I felt lets first make the
> current patch stable. Once all the fields are properly decided and the
> patch gets in then we can easily extend the functionality to shutdown
> and end-of-recovery cases. I have also observed that the timer
> functionality wont work properly in case of shutdown as we are doing
> an immediate checkpoint. So this needs a lot of discussion and I would
> like to handle this on a separate thread.
> ---
>
> > 7) I think you don't need to call checkpoint_progress_start and
> > pgstat_progress_update_param, any other progress reporting function
> > for shutdown and end-of-recovery checkpoint right?
>
> I had included the guards earlier and then removed later based on the
> discussion upthread.
> ---
>
> > 8) Not for all kinds of checkpoints right? pg_stat_progress_checkpoint
> > can't show progress report for shutdown and end-of-recovery
> > checkpoint, I think you need to specify that here in wal.sgml and
> > checkpoint.sgml.
> > + command <command>CHECKPOINT</command>. The checkpointer process running the
> > + checkpoint will report its progress in the
> > + <structname>pg_stat_progress_checkpoint</structname> view. See
> > + <xref linkend="checkpoint-progress-reporting"/> for details.
> >
> > 9) Can you add a test case for pg_stat_progress_checkpoint view? I
> > think it's good to add one. See, below for reference:
> > -- Add a trigger to catch and print the contents of the catalog view
> > -- pg_stat_progress_copy during data insertion. This allows to test
> > -- the validation of some progress reports for COPY FROM where the trigger
> > -- would fire.
> > create function notice_after_tab_progress_reporting() returns trigger AS
> > $$
> > declare report record;
> >
> > 10) Typo: it's not "is happens"
> > + The checkpoint is happens without delays.
> >
> > 11) Can you be specific what are those "some operations" that forced a
> > checkpoint? May be like, basebackup, createdb or something?
> > + The checkpoint is started because some operation forced a checkpoint.
> >
> > 12) Can you be a bit elobartive here who waits? Something like the
> > backend that requested checkpoint will wait until it's completion ....
> > + Wait for completion before returning.
> >
> > 13) "removing unneeded or flushing needed logical rewrite mapping files"
> > + The checkpointer process is currently removing/flushing the logical
> >
> > 14) "old WAL files"
> > + The checkpointer process is currently recycling old XLOG files.
>
> I will take care in the next patch.
>
> Thanks & Regards,
> Nitin Jadhav
>
> On Wed, Mar 2, 2022 at 11:52 PM Bharath Rupireddy
> <[email protected]> wrote:
> >
> > On Wed, Mar 2, 2022 at 4:45 PM Nitin Jadhav
> > <[email protected]> wrote:
> > > > Also, how about special phases for SyncPostCheckpoint(),
> > > > SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
> > > > PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
> > > > it might be increase in future (?)), TruncateSUBTRANS()?
> > >
> > > SyncPreCheckpoint() is just incrementing a counter and
> > > PreallocXlogFiles() currently pre-allocates only 1 WAL file. I feel
> > > there is no need to add any phases for these as of now. We can add in
> > > the future if necessary. Added phases for SyncPostCheckpoint(),
> > > InvalidateObsoleteReplicationSlots() and TruncateSUBTRANS().
> >
> > Okay.
> >
> > > > 10) I'm not sure if it's discussed, how about adding the number of
> > > > snapshot/mapping files so far the checkpoint has processed in file
> > > > processing while loops of
> > > > CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
> > > > be many logical snapshot or mapping files and users may be interested
> > > > in knowing the so-far-processed-file-count.
> > >
> > > I had thought about this while sharing the v1 patch and mentioned my
> > > views upthread. I feel it won't give meaningful progress information
> > > (It can be treated as statistics). Hence not included. Thoughts?
> >
> > Okay. If there are any complaints about it we can always add them later.
> >
> > > > > > As mentioned upthread, there can be multiple backends that request a
> > > > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > > > of backend that are waiting for a new checkpoint.
> > > > >
> > > > > Yeah, you are right. Let's not go that path and store an array of
> > > > > pids. I don't see a strong use-case with the pid of the process
> > > > > requesting checkpoint. If required, we can add it later once the
> > > > > pg_stat_progress_checkpoint view gets in.
> > > >
> > > > I don't think that's really necessary to give the pid list.
> > > >
> > > > If you requested a new checkpoint, it doesn't matter if it's only your backend
> > > > that triggered it, another backend or a few other dozen, the result will be the
> > > > same and you have the information that the request has been seen. We could
> > > > store just a bool for that but having a number instead also gives a bit more
> > > > information and may allow you to detect some broken logic on your client code
> > > > if it keeps increasing.
> > >
> > > It's a good metric to show in the view but the information is not
> > > readily available. Additional code is required to calculate the number
> > > of requests. Is it worth doing that? I feel this can be added later if
> > > required.
> >
> > Yes, we can always add it later if required.
> >
> > > Please find the v4 patch attached and share your thoughts.
> >
> > I reviewed v4 patch, here are my comments:
> >
> > 1) Can we convert below into pgstat_progress_update_multi_param, just
> > to avoid function calls?
> > pgstat_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
> > pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> >
> > 2) Why are we not having special phase for CheckPointReplicationOrigin
> > as it does good bunch of work (writing to disk, XLogFlush,
> > durable_rename) especially when max_replication_slots is large?
> >
> > 3) I don't think "requested" is necessary here as it doesn't add any
> > value or it's not a checkpoint kind or such, you can remove it.
> >
> > 4) s:/'recycling old XLOG files'/'recycling old WAL files'
> > + WHEN 16 THEN 'recycling old XLOG files'
> >
> > 5) Can we place CREATE VIEW pg_stat_progress_checkpoint AS definition
> > next to pg_stat_progress_copy in system_view.sql? It looks like all
> > the progress reporting views are next to each other.
> >
> > 6) How about shutdown and end-of-recovery checkpoint? Are you planning
> > to have an ereport_startup_progress mechanism as 0002?
> >
> > 7) I think you don't need to call checkpoint_progress_start and
> > pgstat_progress_update_param, any other progress reporting function
> > for shutdown and end-of-recovery checkpoint right?
> >
> > 8) Not for all kinds of checkpoints right? pg_stat_progress_checkpoint
> > can't show progress report for shutdown and end-of-recovery
> > checkpoint, I think you need to specify that here in wal.sgml and
> > checkpoint.sgml.
> > + command <command>CHECKPOINT</command>. The checkpointer process running the
> > + checkpoint will report its progress in the
> > + <structname>pg_stat_progress_checkpoint</structname> view. See
> > + <xref linkend="checkpoint-progress-reporting"/> for details.
> >
> > 9) Can you add a test case for pg_stat_progress_checkpoint view? I
> > think it's good to add one. See, below for reference:
> > -- Add a trigger to catch and print the contents of the catalog view
> > -- pg_stat_progress_copy during data insertion. This allows to test
> > -- the validation of some progress reports for COPY FROM where the trigger
> > -- would fire.
> > create function notice_after_tab_progress_reporting() returns trigger AS
> > $$
> > declare report record;
> >
> > 10) Typo: it's not "is happens"
> > + The checkpoint is happens without delays.
> >
> > 11) Can you be specific what are those "some operations" that forced a
> > checkpoint? May be like, basebackup, createdb or something?
> > + The checkpoint is started because some operation forced a checkpoint.
> >
> > 12) Can you be a bit elobartive here who waits? Something like the
> > backend that requested checkpoint will wait until it's completion ....
> > + Wait for completion before returning.
> >
> > 13) "removing unneeded or flushing needed logical rewrite mapping files"
> > + The checkpointer process is currently removing/flushing the logical
> >
> > 14) "old WAL files"
> > + The checkpointer process is currently recycling old XLOG files.
> >
> > Regards,
> > Bharath Rupireddy.
Attachments:
[application/octet-stream] v5-0001-pg_stat_progress_checkpoint-view.patch (38.6K, ../../CAMm1aWaFmbc8QCbcPkG7ROV7gjshamS=sJRJd=oghLJM9uOnpQ@mail.gmail.com/2-v5-0001-pg_stat_progress_checkpoint-view.patch)
download | inline diff:
From 8e1ec8bb49dfe7c753e11ac5c97a82751dc1d1b2 Mon Sep 17 00:00:00 2001
From: Nitin Jadhav <[email protected]>
Date: Tue, 8 Mar 2022 14:36:48 +0000
Subject: [PATCH] pg_stat_progress_checkpoint view
---
doc/src/sgml/monitoring.sgml | 404 +++++++++++++++++++++++++-
doc/src/sgml/ref/checkpoint.sgml | 7 +
doc/src/sgml/wal.sgml | 6 +-
src/backend/access/transam/xlog.c | 102 +++++++
src/backend/catalog/system_views.sql | 59 ++++
src/backend/postmaster/checkpointer.c | 4 +
src/backend/storage/buffer/bufmgr.c | 7 +
src/backend/storage/sync/sync.c | 6 +
src/backend/utils/adt/pgstatfuncs.c | 2 +
src/include/commands/progress.h | 38 +++
src/include/utils/backend_progress.h | 3 +-
src/test/regress/expected/rules.out | 106 +++++++
12 files changed, 741 insertions(+), 3 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 9fb62fec8e..18f4cb4221 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -401,6 +401,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the checkpoint.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -5556,7 +5563,7 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
which support progress reporting are <command>ANALYZE</command>,
<command>CLUSTER</command>,
<command>CREATE INDEX</command>, <command>VACUUM</command>,
- <command>COPY</command>,
+ <command>COPY</command>, <command>CHECKPOINT</command>
and <xref linkend="protocol-replication-base-backup"/> (i.e., replication
command that <xref linkend="app-pgbasebackup"/> issues to take
a base backup).
@@ -6844,6 +6851,401 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>Checkpoint Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the checkpointer process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of the checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>flags</structfield> <type>text</type>
+ </para>
+ <para>
+ Flags of the checkpoint. See <xref linkend="checkpoint-flags"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>next_flags</structfield> <type>text</type>
+ </para>
+ <para>
+ Flags of the next checkpoint. See <xref linkend="checkpoint-flags"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_lsn</structfield> <type>text</type>
+ </para>
+ <para>
+ The checkpoint start location.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_time</structfield> <type>timestamp with time zone</type>
+ </para>
+ <para>
+ Elapsed time of the checkpoint.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal>buffers_total</literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>buffers_total</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>files_total</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-types">
+ <title>Checkpoint Types</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Types</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>checkpoint</literal></entry>
+ <entry>
+ The current operation is checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>restartpoint</literal></entry>
+ <entry>
+ The current operation is restartpoint.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-flags">
+ <title>Checkpoint Flags</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Flags</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>shutdown</literal></entry>
+ <entry>
+ The checkpoint is for shutdown.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>end-of-recovery</literal></entry>
+ <entry>
+ The checkpoint is for end-of-recovery.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>immediate</literal></entry>
+ <entry>
+ The checkpoint happens without delays.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint is started because some operation (for which the
+ checkpoint is necessary) forced a checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>flush all</literal></entry>
+ <entry>
+ The checkpoint flushes all pages, including those belonging to unlogged
+ tables.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wait</literal></entry>
+ <entry>
+ The operations which requested the checkpoint waits for completion
+ before returning.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>requested</literal></entry>
+ <entry>
+ The checkpoint request has been made.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>max_wal_size</literal> is
+ reached.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>checkpoint_timeout</literal>
+ expired.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>Checkpoint Phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The checkpointer process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>getting virtual transaction IDs</literal></entry>
+ <entry>
+ The checkpointer process is getting the virtual transaction IDs that
+ are delaying the checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently flushing all the replication slots
+ to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical replication snapshot files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing all the serialized
+ snapshot files that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mapping files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing unwanted or flushing
+ required logical rewrite mapping files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication origin</literal></entry>
+ <entry>
+ The checkpointer process is currently performing a checkpoint of each
+ replication origin's progress with respect to the replayed remote LSN.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit log pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit log pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit time stamp pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit time stamp pages to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing subtransaction pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing subtransaction pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing multixact pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing multixact pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing predicate lock pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing predicate lock pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The checkpointer process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>processing file sync requests</literal></entry>
+ <entry>
+ The checkpointer process is currently processing file sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The checkpointer process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing post checkpoint cleanup</literal></entry>
+ <entry>
+ The checkpointer process is currently performing post checkpoint cleanup.
+ It removes any lingering files that can be safely removed.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>invalidating replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently invalidating replication slots.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old WAL files</literal></entry>
+ <entry>
+ The checkpointer process is currently recycling old WAL files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>truncating subtransactions</literal></entry>
+ <entry>
+ The checkpointer process is currently removing the subtransaction
+ segments.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The checkpointer process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/doc/src/sgml/ref/checkpoint.sgml b/doc/src/sgml/ref/checkpoint.sgml
index 1cebc03d15..f33db50cfc 100644
--- a/doc/src/sgml/ref/checkpoint.sgml
+++ b/doc/src/sgml/ref/checkpoint.sgml
@@ -56,6 +56,13 @@ CHECKPOINT
the <link linkend="predefined-roles-table"><literal>pg_checkpointer</literal></link>
role can call <command>CHECKPOINT</command>.
</para>
+
+ <para>
+ The checkpointer process running the checkpoint will report its progress
+ in the <structname>pg_stat_progress_checkpoint</structname> view except for
+ the shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
+ </para>
</refsect1>
<refsect1>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 2bb27a8468..8520304abb 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -530,7 +530,11 @@
adjust the <xref linkend="guc-archive-timeout"/> parameter rather than the
checkpoint parameters.)
It is also possible to force a checkpoint by using the SQL
- command <command>CHECKPOINT</command>.
+ command <command>CHECKPOINT</command>. The checkpointer process running the
+ checkpoint will report its progress in the
+ <structname>pg_stat_progress_checkpoint</structname> view except for the
+ shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
</para>
<para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 0d2bd7a357..05dabc6a94 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -65,6 +65,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "commands/progress.h"
#include "common/controldata_utils.h"
#include "executor/instrument.h"
#include "miscadmin.h"
@@ -719,6 +720,8 @@ static void WALInsertLockAcquireExclusive(void);
static void WALInsertLockRelease(void);
static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static void checkpoint_progress_start(int flags, int type);
+
/*
* Insert an XLOG record represented by an already-constructed chain of data
* chunks. This is a low-level routine; to construct the WAL record header
@@ -6419,6 +6422,9 @@ CreateCheckPoint(int flags)
XLogCtl->RedoRecPtr = checkPoint.redo;
SpinLockRelease(&XLogCtl->info_lck);
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_CHECKPOINT);
+
/*
* If enabled, log checkpoint start. We postpone this until now so as not
* to log anything if we decided to skip the checkpoint.
@@ -6501,6 +6507,8 @@ CreateCheckPoint(int flags)
* and we will correctly flush the update below. So we cannot miss any
* xacts we need to wait for.
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS);
vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
if (nvxids > 0)
{
@@ -6604,6 +6612,8 @@ CreateCheckPoint(int flags)
/*
* Let smgr do post-checkpoint cleanup (eg, deleting old files).
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP);
SyncPostCheckpoint();
/*
@@ -6619,6 +6629,9 @@ CreateCheckPoint(int flags)
*/
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
KeepLogSeg(recptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -6629,6 +6642,8 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
@@ -6647,11 +6662,21 @@ CreateCheckPoint(int flags)
* StartupSUBTRANS hasn't been called yet.
*/
if (!RecoveryInProgress())
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -6808,29 +6833,63 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN);
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES);
CheckPointPredicate();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SYNC_FILES);
ProcessSyncRequests();
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -6977,6 +7036,9 @@ CreateRestartPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the restartpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT);
+
if (log_checkpoints)
LogCheckpointStart(flags, true);
@@ -7051,6 +7113,9 @@ CreateRestartPoint(int flags)
replayPtr = GetXLogReplayRecPtr(&replayTLI);
endptr = (receivePtr < replayPtr) ? replayPtr : receivePtr;
KeepLogSeg(endptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -7077,6 +7142,8 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
/*
@@ -7093,11 +7160,20 @@ CreateRestartPoint(int flags)
* this because StartupSUBTRANS hasn't been called yet.
*/
if (EnableHotStandby)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the restartpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -9197,3 +9273,29 @@ SetWalWriterSleeping(bool sleeping)
XLogCtl->WalWriterSleeping = sleeping;
SpinLockRelease(&XLogCtl->info_lck);
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+static void
+checkpoint_progress_start(int flags, int type)
+{
+ const int index[] = {
+ PROGRESS_CHECKPOINT_TYPE,
+ PROGRESS_CHECKPOINT_FLAGS,
+ PROGRESS_CHECKPOINT_LSN,
+ PROGRESS_CHECKPOINT_START_TIMESTAMP,
+ PROGRESS_CHECKPOINT_PHASE
+ };
+ int64 val[5];
+
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+
+ val[0] = type;
+ val[1] = flags;
+ val[2] = RedoRecPtr;
+ val[3] = CheckpointStats.ckpt_start_t;
+ val[4] = PROGRESS_CHECKPOINT_PHASE_INIT;
+
+ pgstat_progress_update_multi_param(5, index, val);
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 40b7bca5a9..bd8583b038 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1228,6 +1228,65 @@ CREATE VIEW pg_stat_progress_copy AS
FROM pg_stat_get_progress_info('COPY') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ CASE S.param1 WHEN 1 THEN 'checkpoint'
+ WHEN 2 THEN 'restartpoint'
+ END AS type,
+ ( CASE WHEN (S.param2 & 1) > 0 THEN 'shutdown ' ELSE '' END ||
+ CASE WHEN (S.param2 & 2) > 0 THEN 'end-of-recovery ' ELSE '' END ||
+ CASE WHEN (S.param2 & 4) > 0 THEN 'immediate ' ELSE '' END ||
+ CASE WHEN (S.param2 & 8) > 0 THEN 'force ' ELSE '' END ||
+ CASE WHEN (S.param2 & 16) > 0 THEN 'flush-all ' ELSE '' END ||
+ CASE WHEN (S.param2 & 32) > 0 THEN 'wait ' ELSE '' END ||
+ CASE WHEN (S.param2 & 128) > 0 THEN 'wal ' ELSE '' END ||
+ CASE WHEN (S.param2 & 256) > 0 THEN 'time ' ELSE '' END
+ ) AS flags,
+ ( CASE WHEN (S.param3 & 1) > 0 THEN 'shutdown ' ELSE '' END ||
+ CASE WHEN (S.param3 & 2) > 0 THEN 'end-of-recovery ' ELSE '' END ||
+ CASE WHEN (S.param3 & 4) > 0 THEN 'immediate ' ELSE '' END ||
+ CASE WHEN (S.param3 & 8) > 0 THEN 'force ' ELSE '' END ||
+ CASE WHEN (S.param3 & 16) > 0 THEN 'flush-all ' ELSE '' END ||
+ CASE WHEN (S.param3 & 32) > 0 THEN 'wait ' ELSE '' END ||
+ CASE WHEN (S.param3 & 128) > 0 THEN 'wal ' ELSE '' END ||
+ CASE WHEN (S.param3 & 256) > 0 THEN 'time ' ELSE '' END
+ ) AS next_flags,
+ ( '0/0'::pg_lsn +
+ ((CASE
+ WHEN S.param4 < 0 THEN pow(2::numeric, 64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ S.param4::numeric)
+ ) AS start_lsn,
+ to_timestamp(946684800 + (S.param5::float8 / 1000000)) AS start_time,
+ CASE S.param6 WHEN 1 THEN 'initializing'
+ WHEN 2 THEN 'getting virtual transaction IDs'
+ WHEN 3 THEN 'checkpointing replication slots'
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'
+ WHEN 6 THEN 'checkpointing replication origin'
+ WHEN 7 THEN 'checkpointing commit log pages'
+ WHEN 8 THEN 'checkpointing commit time stamp pages'
+ WHEN 9 THEN 'checkpointing subtransaction pages'
+ WHEN 10 THEN 'checkpointing multixact pages'
+ WHEN 11 THEN 'checkpointing predicate lock pages'
+ WHEN 12 THEN 'checkpointing buffers'
+ WHEN 13 THEN 'processing file sync requests'
+ WHEN 14 THEN 'performing two phase checkpoint'
+ WHEN 15 THEN 'performing post checkpoint cleanup'
+ WHEN 16 THEN 'invalidating replication slots'
+ WHEN 17 THEN 'recycling old WAL files'
+ WHEN 18 THEN 'truncating subtransactions'
+ WHEN 19 THEN 'finalizing'
+ END AS phase,
+ S.param7 AS buffers_total,
+ S.param8 AS buffers_processed,
+ S.param9 AS buffers_written,
+ S.param10 AS files_total,
+ S.param11 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
+
CREATE VIEW pg_user_mappings AS
SELECT
U.oid AS umid,
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 4488e3a443..9b7441ed40 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -39,6 +39,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogrecovery.h"
+#include "commands/progress.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -662,6 +663,9 @@ ImmediateCheckpointRequested(void)
{
volatile CheckpointerShmemStruct *cps = CheckpointerShmem;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_NEXT_CHECKPOINT_FLAGS,
+ cps->ckpt_flags);
+
/*
* We don't need to acquire the ckpt_lck in this case because we're only
* looking at a single flag bit.
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index f5459c68f8..9663035d7a 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -38,6 +38,7 @@
#include "access/xlogutils.h"
#include "catalog/catalog.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2012,6 +2013,8 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_TOTAL,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2129,6 +2132,8 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2149,6 +2154,8 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.m_buf_written_checkpoints++;
num_written++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index e161d57761..638d3eb781 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -356,6 +357,9 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_TOTAL,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -419,6 +423,8 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index fd993d0d5f..95df730415 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -494,6 +494,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..316886e6f0 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,42 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_TYPE 0
+#define PROGRESS_CHECKPOINT_FLAGS 1
+#define PROGRESS_CHECKPOINT_NEXT_CHECKPOINT_FLAGS 2
+#define PROGRESS_CHECKPOINT_LSN 3
+#define PROGRESS_CHECKPOINT_START_TIMESTAMP 4
+#define PROGRESS_CHECKPOINT_PHASE 5
+#define PROGRESS_CHECKPOINT_BUFFERS_TOTAL 6
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 7
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 8
+#define PROGRESS_CHECKPOINT_FILES_TOTAL 9
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 10
+
+/* Types of checkpoint (as advertised via PROGRESS_CHECKPOINT_TYPE) */
+#define PROGRESS_CHECKPOINT_TYPE_CHECKPOINT 1
+#define PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT 2
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 1
+#define PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS 2
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 3
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 4
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 5
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN 6
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 9
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 10
+#define PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES 11
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 12
+#define PROGRESS_CHECKPOINT_PHASE_SYNC_FILES 13
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 14
+#define PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP 15
+#define PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS 16
+#define PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG 17
+#define PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS 18
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 19
+
#endif
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index ac468568a1..b53a29e372 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1897,6 +1897,112 @@ pg_stat_progress_basebackup| SELECT s.pid,
s.param4 AS tablespaces_total,
s.param5 AS tablespaces_streamed
FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
+pg_stat_progress_checkpoint| SELECT s.pid,
+ CASE s.param1
+ WHEN 1 THEN 'checkpoint'::text
+ WHEN 2 THEN 'restartpoint'::text
+ ELSE NULL::text
+ END AS type,
+ (((((((
+ CASE
+ WHEN ((s.param2 & (1)::bigint) > 0) THEN 'shutdown '::text
+ ELSE ''::text
+ END ||
+ CASE
+ WHEN ((s.param2 & (2)::bigint) > 0) THEN 'end-of-recovery '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (4)::bigint) > 0) THEN 'immediate '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (8)::bigint) > 0) THEN 'force '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (16)::bigint) > 0) THEN 'flush-all '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (32)::bigint) > 0) THEN 'wait '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (128)::bigint) > 0) THEN 'wal '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (256)::bigint) > 0) THEN 'time '::text
+ ELSE ''::text
+ END) AS flags,
+ (((((((
+ CASE
+ WHEN ((s.param3 & (1)::bigint) > 0) THEN 'shutdown '::text
+ ELSE ''::text
+ END ||
+ CASE
+ WHEN ((s.param3 & (2)::bigint) > 0) THEN 'end-of-recovery '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param3 & (4)::bigint) > 0) THEN 'immediate '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param3 & (8)::bigint) > 0) THEN 'force '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param3 & (16)::bigint) > 0) THEN 'flush-all '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param3 & (32)::bigint) > 0) THEN 'wait '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param3 & (128)::bigint) > 0) THEN 'wal '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param3 & (256)::bigint) > 0) THEN 'time '::text
+ ELSE ''::text
+ END) AS next_flags,
+ ('0/0'::pg_lsn + (
+ CASE
+ WHEN (s.param4 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (s.param4)::numeric)) AS start_lsn,
+ to_timestamp(((946684800)::double precision + ((s.param5)::double precision / (1000000)::double precision))) AS start_time,
+ CASE s.param6
+ WHEN 1 THEN 'initializing'::text
+ WHEN 2 THEN 'getting virtual transaction IDs'::text
+ WHEN 3 THEN 'checkpointing replication slots'::text
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'::text
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'::text
+ WHEN 6 THEN 'checkpointing replication origin'::text
+ WHEN 7 THEN 'checkpointing commit log pages'::text
+ WHEN 8 THEN 'checkpointing commit time stamp pages'::text
+ WHEN 9 THEN 'checkpointing subtransaction pages'::text
+ WHEN 10 THEN 'checkpointing multixact pages'::text
+ WHEN 11 THEN 'checkpointing predicate lock pages'::text
+ WHEN 12 THEN 'checkpointing buffers'::text
+ WHEN 13 THEN 'processing file sync requests'::text
+ WHEN 14 THEN 'performing two phase checkpoint'::text
+ WHEN 15 THEN 'performing post checkpoint cleanup'::text
+ WHEN 16 THEN 'invalidating replication slots'::text
+ WHEN 17 THEN 'recycling old WAL files'::text
+ WHEN 18 THEN 'truncating subtransactions'::text
+ WHEN 19 THEN 'finalizing'::text
+ ELSE NULL::text
+ END AS phase,
+ s.param7 AS buffers_total,
+ s.param8 AS buffers_processed,
+ s.param9 AS buffers_written,
+ s.param10 AS files_total,
+ s.param11 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
pg_stat_progress_cluster| SELECT s.pid,
s.datid,
d.datname,
--
2.25.1
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-19 00:15 ` Andres Freund <[email protected]>
2022-04-05 09:45 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Michael Paquier <[email protected]>
2022-04-08 14:52 ` Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint) Matthias van de Meent <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 3 replies; 85+ messages in thread
From: Andres Freund @ 2022-03-19 00:15 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
This is a long thread, sorry for asking if this has been asked before.
On 2022-03-08 20:25:28 +0530, Nitin Jadhav wrote:
> * Sort buffers that need to be written to reduce the likelihood of random
> @@ -2129,6 +2132,8 @@ BufferSync(int flags)
> bufHdr = GetBufferDescriptor(buf_id);
>
> num_processed++;
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
> + num_processed);
>
> /*
> * We don't need to acquire the lock here, because we're only looking
> @@ -2149,6 +2154,8 @@ BufferSync(int flags)
> TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
> PendingCheckpointerStats.m_buf_written_checkpoints++;
> num_written++;
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
> + num_written);
> }
> }
Have you measured the performance effects of this? On fast storage with large
shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
be good to verify that.
> @@ -1897,6 +1897,112 @@ pg_stat_progress_basebackup| SELECT s.pid,
> s.param4 AS tablespaces_total,
> s.param5 AS tablespaces_streamed
> FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
> +pg_stat_progress_checkpoint| SELECT s.pid,
> + CASE s.param1
> + WHEN 1 THEN 'checkpoint'::text
> + WHEN 2 THEN 'restartpoint'::text
> + ELSE NULL::text
> + END AS type,
> + (((((((
> + CASE
> + WHEN ((s.param2 & (1)::bigint) > 0) THEN 'shutdown '::text
> + ELSE ''::text
> + END ||
> + CASE
> + WHEN ((s.param2 & (2)::bigint) > 0) THEN 'end-of-recovery '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param2 & (4)::bigint) > 0) THEN 'immediate '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param2 & (8)::bigint) > 0) THEN 'force '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param2 & (16)::bigint) > 0) THEN 'flush-all '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param2 & (32)::bigint) > 0) THEN 'wait '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param2 & (128)::bigint) > 0) THEN 'wal '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param2 & (256)::bigint) > 0) THEN 'time '::text
> + ELSE ''::text
> + END) AS flags,
> + (((((((
> + CASE
> + WHEN ((s.param3 & (1)::bigint) > 0) THEN 'shutdown '::text
> + ELSE ''::text
> + END ||
> + CASE
> + WHEN ((s.param3 & (2)::bigint) > 0) THEN 'end-of-recovery '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param3 & (4)::bigint) > 0) THEN 'immediate '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param3 & (8)::bigint) > 0) THEN 'force '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param3 & (16)::bigint) > 0) THEN 'flush-all '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param3 & (32)::bigint) > 0) THEN 'wait '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param3 & (128)::bigint) > 0) THEN 'wal '::text
> + ELSE ''::text
> + END) ||
> + CASE
> + WHEN ((s.param3 & (256)::bigint) > 0) THEN 'time '::text
> + ELSE ''::text
> + END) AS next_flags,
> + ('0/0'::pg_lsn + (
> + CASE
> + WHEN (s.param4 < 0) THEN pow((2)::numeric, (64)::numeric)
> + ELSE (0)::numeric
> + END + (s.param4)::numeric)) AS start_lsn,
> + to_timestamp(((946684800)::double precision + ((s.param5)::double precision / (1000000)::double precision))) AS start_time,
> + CASE s.param6
> + WHEN 1 THEN 'initializing'::text
> + WHEN 2 THEN 'getting virtual transaction IDs'::text
> + WHEN 3 THEN 'checkpointing replication slots'::text
> + WHEN 4 THEN 'checkpointing logical replication snapshot files'::text
> + WHEN 5 THEN 'checkpointing logical rewrite mapping files'::text
> + WHEN 6 THEN 'checkpointing replication origin'::text
> + WHEN 7 THEN 'checkpointing commit log pages'::text
> + WHEN 8 THEN 'checkpointing commit time stamp pages'::text
> + WHEN 9 THEN 'checkpointing subtransaction pages'::text
> + WHEN 10 THEN 'checkpointing multixact pages'::text
> + WHEN 11 THEN 'checkpointing predicate lock pages'::text
> + WHEN 12 THEN 'checkpointing buffers'::text
> + WHEN 13 THEN 'processing file sync requests'::text
> + WHEN 14 THEN 'performing two phase checkpoint'::text
> + WHEN 15 THEN 'performing post checkpoint cleanup'::text
> + WHEN 16 THEN 'invalidating replication slots'::text
> + WHEN 17 THEN 'recycling old WAL files'::text
> + WHEN 18 THEN 'truncating subtransactions'::text
> + WHEN 19 THEN 'finalizing'::text
> + ELSE NULL::text
> + END AS phase,
> + s.param7 AS buffers_total,
> + s.param8 AS buffers_processed,
> + s.param9 AS buffers_written,
> + s.param10 AS files_total,
> + s.param11 AS files_synced
> + FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
> pg_stat_progress_cluster| SELECT s.pid,
> s.datid,
> d.datname,
This view is depressingly complicated. Added up the view definitions for
the already existing pg_stat_progress* views add up to a measurable part of
the size of an empty database:
postgres[1160866][1]=# SELECT sum(octet_length(ev_action)), SUM(pg_column_size(ev_action)) FROM pg_rewrite WHERE ev_class::regclass::text LIKE '%progress%';
┌───────┬───────┐
│ sum │ sum │
├───────┼───────┤
│ 97410 │ 19786 │
└───────┴───────┘
(1 row)
and this view looks to be a good bit more complicated than the existing
pg_stat_progress* views.
Indeed:
template1[1165473][1]=# SELECT ev_class::regclass, length(ev_action), pg_column_size(ev_action) FROM pg_rewrite WHERE ev_class::regclass::text LIKE '%progress%' ORDER BY length(ev_action) DESC;
┌───────────────────────────────┬────────┬────────────────┐
│ ev_class │ length │ pg_column_size │
├───────────────────────────────┼────────┼────────────────┤
│ pg_stat_progress_checkpoint │ 43290 │ 5409 │
│ pg_stat_progress_create_index │ 23293 │ 4177 │
│ pg_stat_progress_cluster │ 18390 │ 3704 │
│ pg_stat_progress_analyze │ 16121 │ 3339 │
│ pg_stat_progress_vacuum │ 16076 │ 3392 │
│ pg_stat_progress_copy │ 15124 │ 3080 │
│ pg_stat_progress_basebackup │ 8406 │ 2094 │
└───────────────────────────────┴────────┴────────────────┘
(7 rows)
pg_rewrite without pg_stat_progress_checkpoint: 745472, with: 753664
pg_rewrite is the second biggest relation in an empty database already...
template1[1164827][1]=# SELECT relname, pg_total_relation_size(oid) FROM pg_class WHERE relkind = 'r' ORDER BY 2 DESC LIMIT 5;
┌────────────────┬────────────────────────┐
│ relname │ pg_total_relation_size │
├────────────────┼────────────────────────┤
│ pg_proc │ 1212416 │
│ pg_rewrite │ 745472 │
│ pg_attribute │ 704512 │
│ pg_description │ 630784 │
│ pg_collation │ 409600 │
└────────────────┴────────────────────────┘
(5 rows)
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
@ 2022-04-05 09:45 ` Michael Paquier <[email protected]>
2022-06-13 13:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2 siblings, 1 reply; 85+ messages in thread
From: Michael Paquier @ 2022-04-05 09:45 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Mar 18, 2022 at 05:15:56PM -0700, Andres Freund wrote:
> Have you measured the performance effects of this? On fast storage with large
> shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
> be good to verify that.
I am wondering if we could make the function inlined at some point.
We could also play it safe and only update the counters every N loops
instead.
> This view is depressingly complicated. Added up the view definitions for
> the already existing pg_stat_progress* views add up to a measurable part of
> the size of an empty database:
Yeah. I think that what's proposed could be simplified, and we had
better remove the fields that are not that useful. First, do we have
any need for next_flags? Second, is the start LSN really necessary
for monitoring purposes? Not all the information in the first
parameter is useful, as well. For example "shutdown" will never be
seen as it is not possible to use a session at this stage, no? There
is also no gain in having "immediate", "flush-all", "force" and "wait"
(for this one if the checkpoint is requested the session doing the
work knows this information already).
A last thing is that we may gain in visibility by having more
attributes as an effect of splitting param2. On thing that would make
sense is to track the reason why the checkpoint was triggered
separately (aka wal and time). Should we use a text[] instead to list
all the parameters instead? Using a space-separated list of items is
not intuitive IMO, and callers of this routine will likely parse
that.
Shouldn't we also track the number of files flushed in each sub-step?
In some deployments you could have a large number of 2PC files and
such. We may want more information on such matters.
+ WHEN 3 THEN 'checkpointing replication slots'
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'
+ WHEN 6 THEN 'checkpointing replication origin'
+ WHEN 7 THEN 'checkpointing commit log pages'
+ WHEN 8 THEN 'checkpointing commit time stamp pages'
There is a lot of "checkpointing" here. All those terms could be
shorter without losing their meaning.
This patch still needs some work, so I am marking it as RwF for now.
--
Michael
Attachments:
[application/pgp-signature] signature.asc (833B, ../../YkwPyAoTq%[email protected]/2-signature.asc)
download
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-04-05 09:45 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Michael Paquier <[email protected]>
@ 2022-06-13 13:56 ` Nitin Jadhav <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-06-13 13:56 UTC (permalink / raw)
To: Michael Paquier <[email protected]>; +Cc: Andres Freund <[email protected]>; Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > Have you measured the performance effects of this? On fast storage with large
> > shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
> > be good to verify that.
>
> I am wondering if we could make the function inlined at some point.
> We could also play it safe and only update the counters every N loops
> instead.
The idea looks good but based on the performance numbers shared above,
it is not affecting the performance. So we can use the current
approach as it gives more accurate progress.
---
> > This view is depressingly complicated. Added up the view definitions for
> > the already existing pg_stat_progress* views add up to a measurable part of
> > the size of an empty database:
>
> Yeah. I think that what's proposed could be simplified, and we had
> better remove the fields that are not that useful. First, do we have
> any need for next_flags?
"next_flags" is removed in the v6 patch. Added a "new_requests" field
to get to know whether the current checkpoint is being throttled or
not. Please share your views on this.
---
> Second, is the start LSN really necessary
> for monitoring purposes?
IMO, start LSN is necessary to debug if the checkpoint is taking longer.
---
> Not all the information in the first
> parameter is useful, as well. For example "shutdown" will never be
> seen as it is not possible to use a session at this stage, no?
I understand that "shutdown" and "end-of-recovery" will never be seen
and I have removed it in the v6 patch.
---
> There
> is also no gain in having "immediate", "flush-all", "force" and "wait"
> (for this one if the checkpoint is requested the session doing the
> work knows this information already).
"immediate" is required to understand whether the current checkpoint
is throttled or not. I am not sure about other flags "flush-all",
"force" and "wait". I have just supported all the flags to match the
'checkpoint start' log message. Please share your views. If it is not
really required, I will remove it in the next patch.
---
> A last thing is that we may gain in visibility by having more
> attributes as an effect of splitting param2. On thing that would make
> sense is to track the reason why the checkpoint was triggered
> separately (aka wal and time). Should we use a text[] instead to list
> all the parameters instead? Using a space-separated list of items is
> not intuitive IMO, and callers of this routine will likely parse
> that.
If I understand the above comment correctly, you are saying to
introduce a new field, say "reason" ( possible values are either wal
or time) and the "flags" field will continue to represent the other
flags like "immediate", etc. The idea looks good here. We can
introduce new field "reason" and "flags" field can be renamed to
"throttled" (true/false) if we decide to not support other flags
"flush-all", "force" and "wait".
---
> + WHEN 3 THEN 'checkpointing replication slots'
> + WHEN 4 THEN 'checkpointing logical replication snapshot files'
> + WHEN 5 THEN 'checkpointing logical rewrite mapping files'
> + WHEN 6 THEN 'checkpointing replication origin'
> + WHEN 7 THEN 'checkpointing commit log pages'
> + WHEN 8 THEN 'checkpointing commit time stamp pages'
> There is a lot of "checkpointing" here. All those terms could be
> shorter without losing their meaning.
I will try to make it short in the next patch.
---
Please share your thoughts.
Thanks & Regards,
Nitin Jadhav
On Tue, Apr 5, 2022 at 3:15 PM Michael Paquier <[email protected]> wrote:
>
> On Fri, Mar 18, 2022 at 05:15:56PM -0700, Andres Freund wrote:
> > Have you measured the performance effects of this? On fast storage with large
> > shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
> > be good to verify that.
>
> I am wondering if we could make the function inlined at some point.
> We could also play it safe and only update the counters every N loops
> instead.
>
> > This view is depressingly complicated. Added up the view definitions for
> > the already existing pg_stat_progress* views add up to a measurable part of
> > the size of an empty database:
>
> Yeah. I think that what's proposed could be simplified, and we had
> better remove the fields that are not that useful. First, do we have
> any need for next_flags? Second, is the start LSN really necessary
> for monitoring purposes? Not all the information in the first
> parameter is useful, as well. For example "shutdown" will never be
> seen as it is not possible to use a session at this stage, no? There
> is also no gain in having "immediate", "flush-all", "force" and "wait"
> (for this one if the checkpoint is requested the session doing the
> work knows this information already).
>
> A last thing is that we may gain in visibility by having more
> attributes as an effect of splitting param2. On thing that would make
> sense is to track the reason why the checkpoint was triggered
> separately (aka wal and time). Should we use a text[] instead to list
> all the parameters instead? Using a space-separated list of items is
> not intuitive IMO, and callers of this routine will likely parse
> that.
>
> Shouldn't we also track the number of files flushed in each sub-step?
> In some deployments you could have a large number of 2PC files and
> such. We may want more information on such matters.
>
> + WHEN 3 THEN 'checkpointing replication slots'
> + WHEN 4 THEN 'checkpointing logical replication snapshot files'
> + WHEN 5 THEN 'checkpointing logical rewrite mapping files'
> + WHEN 6 THEN 'checkpointing replication origin'
> + WHEN 7 THEN 'checkpointing commit log pages'
> + WHEN 8 THEN 'checkpointing commit time stamp pages'
> There is a lot of "checkpointing" here. All those terms could be
> shorter without losing their meaning.
>
> This patch still needs some work, so I am marking it as RwF for now.
> --
> Michael
^ permalink raw reply [nested|flat] 85+ messages in thread
* Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
@ 2022-04-08 14:52 ` Matthias van de Meent <[email protected]>
2022-04-08 15:20 ` Re: Size of pg_rewrite Dagfinn Ilmari Mannsåker <[email protected]>
2022-04-08 16:09 ` Re: Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint) Andres Freund <[email protected]>
2 siblings, 2 replies; 85+ messages in thread
From: Matthias van de Meent @ 2022-04-08 14:52 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Sat, 19 Mar 2022 at 01:15, Andres Freund <[email protected]> wrote:
> pg_rewrite without pg_stat_progress_checkpoint: 745472, with: 753664
>
> pg_rewrite is the second biggest relation in an empty database already...
Yeah, that's not great. Thanks for nerd-sniping me into looking into
how views and pg_rewrite rules work, that was very interesting and I
learned quite a lot.
# Immediately potential, limited to progress views
I noticed that the CASE-WHEN (used in translating progress stage index
to stage names) in those progress reporting views can be more
efficiently described (althoug with slightly worse behaviour around
undefined values) using text array lookups (as attached). That
resulted in somewhat smaller rewrite entries for the progress views
(toast compression was good old pglz):
template1=# SELECT sum(octet_length(ev_action)),
SUM(pg_column_size(ev_action)) FROM pg_rewrite WHERE
ev_class::regclass::text LIKE '%progress%';
master:
sum | sum
-------+-------
97277 | 19956
patched:
sum | sum
-------+-------
77069 | 18417
So this seems like a nice improvement of 20% uncompressed / 7% compressed.
I tested various cases of phase number to text translations: `CASE ..
WHEN`; `(ARRAY[]::text[])[index]` and `('{}'::text[])[index]`. See
results below:
postgres=# create or replace view arrayliteral_view as select
(ARRAY['a','b','c','d','e','f']::text[])[index] as name from tst
s(index);
CREATE INDEX
postgres=# create or replace view stringcast_view as select
('{a,b,c,d,e,f}'::text[])[index] as name from tst s(index);
CREATE INDEX
postgres=# create or replace view split_stringcast_view as select
(('{a,b,' || 'c,d,e,f}')::text[])[index] as name from tst s(index);
CREATE VIEW
postgres=# create or replace view case_view as select case index when
0 then 'a' when 1 then 'b' when 2 then 'c' when 3 then 'd' when 4 then
'e' when 5 then 'f' end as name from tst s(index);
CREATE INDEX
postgres=# postgres=# select ev_class::regclass::text,
octet_length(ev_action), pg_column_size(ev_action) from pg_rewrite
where ev_class in ('arrayliteral_view'::regclass::oid,
'case_view'::regclass::oid, 'split_stringcast_view'::regclass::oid,
'stringcast_view'::regclass::oid);
ev_class | octet_length | pg_column_size
-----------------------+--------------+----------------
arrayliteral_view | 3311 | 1322
stringcast_view | 2610 | 1257
case_view | 5170 | 1412
split_stringcast_view | 2847 | 1350
It seems to me that we could consider replacing the CASE statements
with array literals and lookups if we really value our template
database size. But, as text literal concatenations don't seem to get
constant folded before storing them in the rules table, this rewrite
of the views would result in long lines in the system_views.sql file,
or we'd have to deal with the additional overhead of the append
operator and cast nodes.
# Future work; nodeToString / readNode, all rewrite rules
Additionally, we might want to consider other changes like default (or
empty value) elision in nodeToString, if that is considered a
reasonable option and if we really want to reduce the size of the
pg_rewrite table.
I think a lot of space can be recovered from that: A manual removal of
what seemed to be fields with default values (and the removal of all
query location related fields) in the current definition of
pg_stat_progress_create_index reduces its uncompressed size from
23226B raw and 4204B compressed to 13821B raw and 2784B compressed,
for an on-disk space saving of 33% for this view's ev_action.
Do note, however, that that would add significant branching in the
nodeToString and readNode code, which might slow down that code
significantly. I'm not planning on working on that; but in my opinion
that is a viable path to reducing the size of new database catalogs.
-Matthias
PS. attached patch is not to be considered complete - it is a minimal
example of the array literal form. It fails regression tests because I
didn't bother updating or including the regression tests on system
views.
Index: src/backend/catalog/system_views.sql
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
--- a/src/backend/catalog/system_views.sql (revision 32723e5fabcc7db1bf4e897baaf0d251b500c1dc)
+++ b/src/backend/catalog/system_views.sql (date 1649160138886)
@@ -1120,13 +1120,7 @@
SELECT
S.pid AS pid, S.datid AS datid, D.datname AS datname,
CAST(S.relid AS oid) AS relid,
- CASE S.param1 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'acquiring sample rows'
- WHEN 2 THEN 'acquiring inherited sample rows'
- WHEN 3 THEN 'computing statistics'
- WHEN 4 THEN 'computing extended statistics'
- WHEN 5 THEN 'finalizing analyze'
- END AS phase,
+ (('{initializing,acquiring sample rows,acquiring inherited sample rows,computing statistics,computing extended statistics,finalizing analyze}')::text[])[S.param1] AS phase,
S.param2 AS sample_blks_total,
S.param3 AS sample_blks_scanned,
S.param4 AS ext_stats_total,
@@ -1141,14 +1135,8 @@
SELECT
S.pid AS pid, S.datid AS datid, D.datname AS datname,
S.relid AS relid,
- CASE S.param1 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'scanning heap'
- WHEN 2 THEN 'vacuuming indexes'
- WHEN 3 THEN 'vacuuming heap'
- WHEN 4 THEN 'cleaning up indexes'
- WHEN 5 THEN 'truncating heap'
- WHEN 6 THEN 'performing final cleanup'
- END AS phase,
+ (('{initializing,scanning heap,vacuuming indexes,vacuuming heap,cleaning up indexes,truncating heap,performing final cleanup}')::text[]
+ )[S.param1] AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
@@ -1161,18 +1149,8 @@
S.datid AS datid,
D.datname AS datname,
S.relid AS relid,
- CASE S.param1 WHEN 1 THEN 'CLUSTER'
- WHEN 2 THEN 'VACUUM FULL'
- END AS command,
- CASE S.param2 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'seq scanning heap'
- WHEN 2 THEN 'index scanning heap'
- WHEN 3 THEN 'sorting tuples'
- WHEN 4 THEN 'writing new heap'
- WHEN 5 THEN 'swapping relation files'
- WHEN 6 THEN 'rebuilding index'
- WHEN 7 THEN 'performing final cleanup'
- END AS phase,
+ ('{NULL,CLUSTER,VACUUM FULL}'::text[])[S.param1] AS command,
+ (('{initializing,initializing,seq scanning heap,index scanning heap,sorting tuples,writing new heap,swapping relation files,rebuilding index,performing final cleanup}')::text[])[S.param2] AS phase,
CAST(S.param3 AS oid) AS cluster_index_relid,
S.param4 AS heap_tuples_scanned,
S.param5 AS heap_tuples_written,
@@ -1187,24 +1165,12 @@
S.pid AS pid, S.datid AS datid, D.datname AS datname,
S.relid AS relid,
CAST(S.param7 AS oid) AS index_relid,
- CASE S.param1 WHEN 1 THEN 'CREATE INDEX'
- WHEN 2 THEN 'CREATE INDEX CONCURRENTLY'
- WHEN 3 THEN 'REINDEX'
- WHEN 4 THEN 'REINDEX CONCURRENTLY'
- END AS command,
- CASE S.param10 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'waiting for writers before build'
- WHEN 2 THEN 'building index' ||
+ ('{NULL,CREATE INDEX,CREATE INDEX CONCURRENTLY,REINDEX,REINDEX CONCURRENTLY}'::text[])[S.param1] AS command,
+ CASE S.param10 WHEN 2 THEN 'building index' ||
COALESCE((': ' || pg_indexam_progress_phasename(S.param9::oid, S.param11)),
'')
- WHEN 3 THEN 'waiting for writers before validation'
- WHEN 4 THEN 'index validation: scanning index'
- WHEN 5 THEN 'index validation: sorting tuples'
- WHEN 6 THEN 'index validation: scanning table'
- WHEN 7 THEN 'waiting for old snapshots'
- WHEN 8 THEN 'waiting for readers before marking dead'
- WHEN 9 THEN 'waiting for readers before dropping'
- END as phase,
+ ELSE ('{NULL,initializing,waiting for writers before build,NULL,waiting for writers before validation,index validation: scanning index,index validation: sorting tuples,index validation: scanning table,waiting for old snapshots,waiting for readers before marking dead,waiting for readers before dropping}'::text[])[S.param10]
+ END AS phase,
S.param4 AS lockers_total,
S.param5 AS lockers_done,
S.param6 AS current_locker_pid,
@@ -1220,13 +1186,8 @@
CREATE VIEW pg_stat_progress_basebackup AS
SELECT
S.pid AS pid,
- CASE S.param1 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'waiting for checkpoint to finish'
- WHEN 2 THEN 'estimating backup size'
- WHEN 3 THEN 'streaming database files'
- WHEN 4 THEN 'waiting for wal archiving to finish'
- WHEN 5 THEN 'transferring wal files'
- END AS phase,
+ ('{NULL,initializing,waiting for checkpoint to finish,estimating backup size,streaming database files,waiting for wal archiving to finish,transferring wal files}'::text[]
+ )[S.param1] AS phase,
CASE S.param2 WHEN -1 THEN NULL ELSE S.param2 END AS backup_total,
S.param3 AS backup_streamed,
S.param4 AS tablespaces_total,
@@ -1238,14 +1199,8 @@
SELECT
S.pid AS pid, S.datid AS datid, D.datname AS datname,
S.relid AS relid,
- CASE S.param5 WHEN 1 THEN 'COPY FROM'
- WHEN 2 THEN 'COPY TO'
- END AS command,
- CASE S.param6 WHEN 1 THEN 'FILE'
- WHEN 2 THEN 'PROGRAM'
- WHEN 3 THEN 'PIPE'
- WHEN 4 THEN 'CALLBACK'
- END AS "type",
+ ('{NULL,COPY FROM,COPY TO}'::text[])[S.param5] AS command,
+ ('{NULL,FILE,PROGRAM,PIPE,CALLBACK}'::text[])[S.param6] AS "type",
S.param1 AS bytes_processed,
S.param2 AS bytes_total,
S.param3 AS tuples_processed,
Attachments:
[text/plain] v0-Replace-system-view-CASE-WHEN-with-array-lookups.patch.txt (6.7K, ../../CAEze2WgGexDM63dOvndLdAWwA6uSmSsc97jmrCuNmrF1JEDK7w@mail.gmail.com/2-v0-Replace-system-view-CASE-WHEN-with-array-lookups.patch.txt)
download | inline diff:
Index: src/backend/catalog/system_views.sql
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
--- a/src/backend/catalog/system_views.sql (revision 32723e5fabcc7db1bf4e897baaf0d251b500c1dc)
+++ b/src/backend/catalog/system_views.sql (date 1649160138886)
@@ -1120,13 +1120,7 @@
SELECT
S.pid AS pid, S.datid AS datid, D.datname AS datname,
CAST(S.relid AS oid) AS relid,
- CASE S.param1 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'acquiring sample rows'
- WHEN 2 THEN 'acquiring inherited sample rows'
- WHEN 3 THEN 'computing statistics'
- WHEN 4 THEN 'computing extended statistics'
- WHEN 5 THEN 'finalizing analyze'
- END AS phase,
+ (('{initializing,acquiring sample rows,acquiring inherited sample rows,computing statistics,computing extended statistics,finalizing analyze}')::text[])[S.param1] AS phase,
S.param2 AS sample_blks_total,
S.param3 AS sample_blks_scanned,
S.param4 AS ext_stats_total,
@@ -1141,14 +1135,8 @@
SELECT
S.pid AS pid, S.datid AS datid, D.datname AS datname,
S.relid AS relid,
- CASE S.param1 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'scanning heap'
- WHEN 2 THEN 'vacuuming indexes'
- WHEN 3 THEN 'vacuuming heap'
- WHEN 4 THEN 'cleaning up indexes'
- WHEN 5 THEN 'truncating heap'
- WHEN 6 THEN 'performing final cleanup'
- END AS phase,
+ (('{initializing,scanning heap,vacuuming indexes,vacuuming heap,cleaning up indexes,truncating heap,performing final cleanup}')::text[]
+ )[S.param1] AS phase,
S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned,
S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count,
S.param6 AS max_dead_tuples, S.param7 AS num_dead_tuples
@@ -1161,18 +1149,8 @@
S.datid AS datid,
D.datname AS datname,
S.relid AS relid,
- CASE S.param1 WHEN 1 THEN 'CLUSTER'
- WHEN 2 THEN 'VACUUM FULL'
- END AS command,
- CASE S.param2 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'seq scanning heap'
- WHEN 2 THEN 'index scanning heap'
- WHEN 3 THEN 'sorting tuples'
- WHEN 4 THEN 'writing new heap'
- WHEN 5 THEN 'swapping relation files'
- WHEN 6 THEN 'rebuilding index'
- WHEN 7 THEN 'performing final cleanup'
- END AS phase,
+ ('{NULL,CLUSTER,VACUUM FULL}'::text[])[S.param1] AS command,
+ (('{initializing,initializing,seq scanning heap,index scanning heap,sorting tuples,writing new heap,swapping relation files,rebuilding index,performing final cleanup}')::text[])[S.param2] AS phase,
CAST(S.param3 AS oid) AS cluster_index_relid,
S.param4 AS heap_tuples_scanned,
S.param5 AS heap_tuples_written,
@@ -1187,24 +1165,12 @@
S.pid AS pid, S.datid AS datid, D.datname AS datname,
S.relid AS relid,
CAST(S.param7 AS oid) AS index_relid,
- CASE S.param1 WHEN 1 THEN 'CREATE INDEX'
- WHEN 2 THEN 'CREATE INDEX CONCURRENTLY'
- WHEN 3 THEN 'REINDEX'
- WHEN 4 THEN 'REINDEX CONCURRENTLY'
- END AS command,
- CASE S.param10 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'waiting for writers before build'
- WHEN 2 THEN 'building index' ||
+ ('{NULL,CREATE INDEX,CREATE INDEX CONCURRENTLY,REINDEX,REINDEX CONCURRENTLY}'::text[])[S.param1] AS command,
+ CASE S.param10 WHEN 2 THEN 'building index' ||
COALESCE((': ' || pg_indexam_progress_phasename(S.param9::oid, S.param11)),
'')
- WHEN 3 THEN 'waiting for writers before validation'
- WHEN 4 THEN 'index validation: scanning index'
- WHEN 5 THEN 'index validation: sorting tuples'
- WHEN 6 THEN 'index validation: scanning table'
- WHEN 7 THEN 'waiting for old snapshots'
- WHEN 8 THEN 'waiting for readers before marking dead'
- WHEN 9 THEN 'waiting for readers before dropping'
- END as phase,
+ ELSE ('{NULL,initializing,waiting for writers before build,NULL,waiting for writers before validation,index validation: scanning index,index validation: sorting tuples,index validation: scanning table,waiting for old snapshots,waiting for readers before marking dead,waiting for readers before dropping}'::text[])[S.param10]
+ END AS phase,
S.param4 AS lockers_total,
S.param5 AS lockers_done,
S.param6 AS current_locker_pid,
@@ -1220,13 +1186,8 @@
CREATE VIEW pg_stat_progress_basebackup AS
SELECT
S.pid AS pid,
- CASE S.param1 WHEN 0 THEN 'initializing'
- WHEN 1 THEN 'waiting for checkpoint to finish'
- WHEN 2 THEN 'estimating backup size'
- WHEN 3 THEN 'streaming database files'
- WHEN 4 THEN 'waiting for wal archiving to finish'
- WHEN 5 THEN 'transferring wal files'
- END AS phase,
+ ('{NULL,initializing,waiting for checkpoint to finish,estimating backup size,streaming database files,waiting for wal archiving to finish,transferring wal files}'::text[]
+ )[S.param1] AS phase,
CASE S.param2 WHEN -1 THEN NULL ELSE S.param2 END AS backup_total,
S.param3 AS backup_streamed,
S.param4 AS tablespaces_total,
@@ -1238,14 +1199,8 @@
SELECT
S.pid AS pid, S.datid AS datid, D.datname AS datname,
S.relid AS relid,
- CASE S.param5 WHEN 1 THEN 'COPY FROM'
- WHEN 2 THEN 'COPY TO'
- END AS command,
- CASE S.param6 WHEN 1 THEN 'FILE'
- WHEN 2 THEN 'PROGRAM'
- WHEN 3 THEN 'PIPE'
- WHEN 4 THEN 'CALLBACK'
- END AS "type",
+ ('{NULL,COPY FROM,COPY TO}'::text[])[S.param5] AS command,
+ ('{NULL,FILE,PROGRAM,PIPE,CALLBACK}'::text[])[S.param6] AS "type",
S.param1 AS bytes_processed,
S.param2 AS bytes_total,
S.param3 AS tuples_processed,
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Size of pg_rewrite
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-04-08 14:52 ` Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint) Matthias van de Meent <[email protected]>
@ 2022-04-08 15:20 ` Dagfinn Ilmari Mannsåker <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Dagfinn Ilmari Mannsåker @ 2022-04-08 15:20 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Andres Freund <[email protected]>; Nitin Jadhav <[email protected]>; Bharath Rupireddy <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Matthias van de Meent <[email protected]> writes:
> But, as text literal concatenations don't seem to get constant folded
> before storing them in the rules table, this rewrite of the views
> would result in long lines in the system_views.sql file, or we'd have
> to deal with the additional overhead of the append operator and cast
> nodes.
There is no need to use the concatenation operator to split array
constants across multiple lines. Newlines are fine either inside the
string (between array elements), or between two string string literals
(which become one string constant at parse time).
https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS
ilmari@[local]:5432 ~=# select
ilmari@[local]:5432 ~-# '{foo,
ilmari@[local]:5432 ~'# bar}'::text[],
ilmari@[local]:5432 ~-# '{bar,'
ilmari@[local]:5432 ~-# 'baz}'::text[];
┌───────────┬───────────┐
│ text │ text │
├───────────┼───────────┤
│ {foo,bar} │ {bar,baz} │
└───────────┴───────────┘
(1 row)
- ilmari
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-04-08 14:52 ` Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint) Matthias van de Meent <[email protected]>
@ 2022-04-08 16:09 ` Andres Freund <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Andres Freund @ 2022-04-08 16:09 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Bharath Rupireddy <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On April 8, 2022 7:52:07 AM PDT, Matthias van de Meent <[email protected]> wrote:
>On Sat, 19 Mar 2022 at 01:15, Andres Freund <[email protected]> wrote:
>> pg_rewrite without pg_stat_progress_checkpoint: 745472, with: 753664
>>
>> pg_rewrite is the second biggest relation in an empty database already...
>
>Yeah, that's not great. Thanks for nerd-sniping me into looking into
>how views and pg_rewrite rules work, that was very interesting and I
>learned quite a lot.
Thanks for looking!
># Immediately potential, limited to progress views
>
>I noticed that the CASE-WHEN (used in translating progress stage index
>to stage names) in those progress reporting views can be more
>efficiently described (althoug with slightly worse behaviour around
>undefined values) using text array lookups (as attached). That
>resulted in somewhat smaller rewrite entries for the progress views
>(toast compression was good old pglz):
>
>template1=# SELECT sum(octet_length(ev_action)),
>SUM(pg_column_size(ev_action)) FROM pg_rewrite WHERE
>ev_class::regclass::text LIKE '%progress%';
>
>master:
> sum | sum
>-------+-------
> 97277 | 19956
>patched:
> sum | sum
>-------+-------
> 77069 | 18417
>
>So this seems like a nice improvement of 20% uncompressed / 7% compressed.
>
>I tested various cases of phase number to text translations: `CASE ..
>WHEN`; `(ARRAY[]::text[])[index]` and `('{}'::text[])[index]`. See
>results below:
>
>postgres=# create or replace view arrayliteral_view as select
>(ARRAY['a','b','c','d','e','f']::text[])[index] as name from tst
>s(index);
>CREATE INDEX
>postgres=# create or replace view stringcast_view as select
>('{a,b,c,d,e,f}'::text[])[index] as name from tst s(index);
>CREATE INDEX
>postgres=# create or replace view split_stringcast_view as select
>(('{a,b,' || 'c,d,e,f}')::text[])[index] as name from tst s(index);
>CREATE VIEW
>postgres=# create or replace view case_view as select case index when
>0 then 'a' when 1 then 'b' when 2 then 'c' when 3 then 'd' when 4 then
>'e' when 5 then 'f' end as name from tst s(index);
>CREATE INDEX
>
>
>postgres=# postgres=# select ev_class::regclass::text,
>octet_length(ev_action), pg_column_size(ev_action) from pg_rewrite
>where ev_class in ('arrayliteral_view'::regclass::oid,
>'case_view'::regclass::oid, 'split_stringcast_view'::regclass::oid,
>'stringcast_view'::regclass::oid);
> ev_class | octet_length | pg_column_size
>-----------------------+--------------+----------------
> arrayliteral_view | 3311 | 1322
> stringcast_view | 2610 | 1257
> case_view | 5170 | 1412
> split_stringcast_view | 2847 | 1350
>
>It seems to me that we could consider replacing the CASE statements
>with array literals and lookups if we really value our template
>database size. But, as text literal concatenations don't seem to get
>constant folded before storing them in the rules table, this rewrite
>of the views would result in long lines in the system_views.sql file,
>or we'd have to deal with the additional overhead of the append
>operator and cast nodes.
My inclination is that the mapping functions should be c functions. There's really no point in doing it in SQL and it comes at a noticable price. And, if done in C, we can fix mistakes in minor releases, which we can't in SQL.
># Future work; nodeToString / readNode, all rewrite rules
>
>Additionally, we might want to consider other changes like default (or
>empty value) elision in nodeToString, if that is considered a
>reasonable option and if we really want to reduce the size of the
>pg_rewrite table.
>
>I think a lot of space can be recovered from that: A manual removal of
>what seemed to be fields with default values (and the removal of all
>query location related fields) in the current definition of
>pg_stat_progress_create_index reduces its uncompressed size from
>23226B raw and 4204B compressed to 13821B raw and 2784B compressed,
>for an on-disk space saving of 33% for this view's ev_action.
>
>Do note, however, that that would add significant branching in the
>nodeToString and readNode code, which might slow down that code
>significantly. I'm not planning on working on that; but in my opinion
>that is a viable path to reducing the size of new database catalogs.
We should definitely be careful about that. I do agree that there's a lot of efficiency to be gained in the serialization format. Once we have the automatic node func generation in place, we could have one representation for human consumption, and one for density...
Andres
--
Sent from my Android device with K-9 Mail. Please excuse my brevity.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
@ 2022-06-13 13:38 ` Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-06-13 13:38 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> Have you measured the performance effects of this? On fast storage with large
> shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
> be good to verify that.
To understand the performance effects of the above, I have taken the
average of five checkpoints with the patch and without the patch in my
environment. Here are the results.
With patch: 269.65 s
Without patch: 269.60 s
It looks fine. Please share your views.
> This view is depressingly complicated. Added up the view definitions for
> the already existing pg_stat_progress* views add up to a measurable part of
> the size of an empty database:
Thank you so much for sharing the detailed analysis. We can remove a
few fields which are not so important to make it simple.
Thanks & Regards,
Nitin Jadhav
On Sat, Mar 19, 2022 at 5:45 AM Andres Freund <[email protected]> wrote:
>
> Hi,
>
> This is a long thread, sorry for asking if this has been asked before.
>
> On 2022-03-08 20:25:28 +0530, Nitin Jadhav wrote:
> > * Sort buffers that need to be written to reduce the likelihood of random
> > @@ -2129,6 +2132,8 @@ BufferSync(int flags)
> > bufHdr = GetBufferDescriptor(buf_id);
> >
> > num_processed++;
> > + pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
> > + num_processed);
> >
> > /*
> > * We don't need to acquire the lock here, because we're only looking
> > @@ -2149,6 +2154,8 @@ BufferSync(int flags)
> > TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
> > PendingCheckpointerStats.m_buf_written_checkpoints++;
> > num_written++;
> > + pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
> > + num_written);
> > }
> > }
>
> Have you measured the performance effects of this? On fast storage with large
> shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
> be good to verify that.
>
>
> > @@ -1897,6 +1897,112 @@ pg_stat_progress_basebackup| SELECT s.pid,
> > s.param4 AS tablespaces_total,
> > s.param5 AS tablespaces_streamed
> > FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
> > +pg_stat_progress_checkpoint| SELECT s.pid,
> > + CASE s.param1
> > + WHEN 1 THEN 'checkpoint'::text
> > + WHEN 2 THEN 'restartpoint'::text
> > + ELSE NULL::text
> > + END AS type,
> > + (((((((
> > + CASE
> > + WHEN ((s.param2 & (1)::bigint) > 0) THEN 'shutdown '::text
> > + ELSE ''::text
> > + END ||
> > + CASE
> > + WHEN ((s.param2 & (2)::bigint) > 0) THEN 'end-of-recovery '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param2 & (4)::bigint) > 0) THEN 'immediate '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param2 & (8)::bigint) > 0) THEN 'force '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param2 & (16)::bigint) > 0) THEN 'flush-all '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param2 & (32)::bigint) > 0) THEN 'wait '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param2 & (128)::bigint) > 0) THEN 'wal '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param2 & (256)::bigint) > 0) THEN 'time '::text
> > + ELSE ''::text
> > + END) AS flags,
> > + (((((((
> > + CASE
> > + WHEN ((s.param3 & (1)::bigint) > 0) THEN 'shutdown '::text
> > + ELSE ''::text
> > + END ||
> > + CASE
> > + WHEN ((s.param3 & (2)::bigint) > 0) THEN 'end-of-recovery '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param3 & (4)::bigint) > 0) THEN 'immediate '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param3 & (8)::bigint) > 0) THEN 'force '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param3 & (16)::bigint) > 0) THEN 'flush-all '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param3 & (32)::bigint) > 0) THEN 'wait '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param3 & (128)::bigint) > 0) THEN 'wal '::text
> > + ELSE ''::text
> > + END) ||
> > + CASE
> > + WHEN ((s.param3 & (256)::bigint) > 0) THEN 'time '::text
> > + ELSE ''::text
> > + END) AS next_flags,
> > + ('0/0'::pg_lsn + (
> > + CASE
> > + WHEN (s.param4 < 0) THEN pow((2)::numeric, (64)::numeric)
> > + ELSE (0)::numeric
> > + END + (s.param4)::numeric)) AS start_lsn,
> > + to_timestamp(((946684800)::double precision + ((s.param5)::double precision / (1000000)::double precision))) AS start_time,
> > + CASE s.param6
> > + WHEN 1 THEN 'initializing'::text
> > + WHEN 2 THEN 'getting virtual transaction IDs'::text
> > + WHEN 3 THEN 'checkpointing replication slots'::text
> > + WHEN 4 THEN 'checkpointing logical replication snapshot files'::text
> > + WHEN 5 THEN 'checkpointing logical rewrite mapping files'::text
> > + WHEN 6 THEN 'checkpointing replication origin'::text
> > + WHEN 7 THEN 'checkpointing commit log pages'::text
> > + WHEN 8 THEN 'checkpointing commit time stamp pages'::text
> > + WHEN 9 THEN 'checkpointing subtransaction pages'::text
> > + WHEN 10 THEN 'checkpointing multixact pages'::text
> > + WHEN 11 THEN 'checkpointing predicate lock pages'::text
> > + WHEN 12 THEN 'checkpointing buffers'::text
> > + WHEN 13 THEN 'processing file sync requests'::text
> > + WHEN 14 THEN 'performing two phase checkpoint'::text
> > + WHEN 15 THEN 'performing post checkpoint cleanup'::text
> > + WHEN 16 THEN 'invalidating replication slots'::text
> > + WHEN 17 THEN 'recycling old WAL files'::text
> > + WHEN 18 THEN 'truncating subtransactions'::text
> > + WHEN 19 THEN 'finalizing'::text
> > + ELSE NULL::text
> > + END AS phase,
> > + s.param7 AS buffers_total,
> > + s.param8 AS buffers_processed,
> > + s.param9 AS buffers_written,
> > + s.param10 AS files_total,
> > + s.param11 AS files_synced
> > + FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
> > pg_stat_progress_cluster| SELECT s.pid,
> > s.datid,
> > d.datname,
>
> This view is depressingly complicated. Added up the view definitions for
> the already existing pg_stat_progress* views add up to a measurable part of
> the size of an empty database:
>
> postgres[1160866][1]=# SELECT sum(octet_length(ev_action)), SUM(pg_column_size(ev_action)) FROM pg_rewrite WHERE ev_class::regclass::text LIKE '%progress%';
> ┌───────┬───────┐
> │ sum │ sum │
> ├───────┼───────┤
> │ 97410 │ 19786 │
> └───────┴───────┘
> (1 row)
>
> and this view looks to be a good bit more complicated than the existing
> pg_stat_progress* views.
>
> Indeed:
> template1[1165473][1]=# SELECT ev_class::regclass, length(ev_action), pg_column_size(ev_action) FROM pg_rewrite WHERE ev_class::regclass::text LIKE '%progress%' ORDER BY length(ev_action) DESC;
> ┌───────────────────────────────┬────────┬────────────────┐
> │ ev_class │ length │ pg_column_size │
> ├───────────────────────────────┼────────┼────────────────┤
> │ pg_stat_progress_checkpoint │ 43290 │ 5409 │
> │ pg_stat_progress_create_index │ 23293 │ 4177 │
> │ pg_stat_progress_cluster │ 18390 │ 3704 │
> │ pg_stat_progress_analyze │ 16121 │ 3339 │
> │ pg_stat_progress_vacuum │ 16076 │ 3392 │
> │ pg_stat_progress_copy │ 15124 │ 3080 │
> │ pg_stat_progress_basebackup │ 8406 │ 2094 │
> └───────────────────────────────┴────────┴────────────────┘
> (7 rows)
>
> pg_rewrite without pg_stat_progress_checkpoint: 745472, with: 753664
>
>
> pg_rewrite is the second biggest relation in an empty database already...
>
> template1[1164827][1]=# SELECT relname, pg_total_relation_size(oid) FROM pg_class WHERE relkind = 'r' ORDER BY 2 DESC LIMIT 5;
> ┌────────────────┬────────────────────────┐
> │ relname │ pg_total_relation_size │
> ├────────────────┼────────────────────────┤
> │ pg_proc │ 1212416 │
> │ pg_rewrite │ 745472 │
> │ pg_attribute │ 704512 │
> │ pg_description │ 630784 │
> │ pg_collation │ 409600 │
> └────────────────┴────────────────────────┘
> (5 rows)
>
> Greetings,
>
> Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-07-07 00:04 ` Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Andres Freund @ 2022-07-07 00:04 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 2022-06-13 19:08:35 +0530, Nitin Jadhav wrote:
> > Have you measured the performance effects of this? On fast storage with large
> > shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
> > be good to verify that.
>
> To understand the performance effects of the above, I have taken the
> average of five checkpoints with the patch and without the patch in my
> environment. Here are the results.
> With patch: 269.65 s
> Without patch: 269.60 s
Those look like timed checkpoints - if the checkpoints are sleeping a
part of the time, you're not going to see any potential overhead.
To see whether this has an effect you'd have to make sure there's a
certain number of dirty buffers (e.g. by doing CREATE TABLE AS
some_query) and then do a manual checkpoint and time how long that
times.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
@ 2022-07-28 09:38 ` Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-07-28 09:38 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > To understand the performance effects of the above, I have taken the
> > average of five checkpoints with the patch and without the patch in my
> > environment. Here are the results.
> > With patch: 269.65 s
> > Without patch: 269.60 s
>
> Those look like timed checkpoints - if the checkpoints are sleeping a
> part of the time, you're not going to see any potential overhead.
Yes. The above data is collected from timed checkpoints.
create table t1(a int);
insert into t1 select * from generate_series(1,10000000);
I generated a lot of data by using the above queries which would in
turn trigger the checkpoint (wal).
---
> To see whether this has an effect you'd have to make sure there's a
> certain number of dirty buffers (e.g. by doing CREATE TABLE AS
> some_query) and then do a manual checkpoint and time how long that
> times.
For this case I have generated data by using below queries.
create table t1(a int);
insert into t1 select * from generate_series(1,8000000);
This does not trigger the checkpoint automatically. I have issued the
CHECKPOINT manually and measured the performance by considering an
average of 5 checkpoints. Here are the details.
With patch: 2.457 s
Without patch: 2.334 s
Please share your thoughts.
Thanks & Regards,
Nitin Jadhav
On Thu, Jul 7, 2022 at 5:34 AM Andres Freund <[email protected]> wrote:
>
> Hi,
>
> On 2022-06-13 19:08:35 +0530, Nitin Jadhav wrote:
> > > Have you measured the performance effects of this? On fast storage with large
> > > shared_buffers I've seen these loops in profiles. It's probably fine, but it'd
> > > be good to verify that.
> >
> > To understand the performance effects of the above, I have taken the
> > average of five checkpoints with the patch and without the patch in my
> > environment. Here are the results.
> > With patch: 269.65 s
> > Without patch: 269.60 s
>
> Those look like timed checkpoints - if the checkpoints are sleeping a
> part of the time, you're not going to see any potential overhead.
>
> To see whether this has an effect you'd have to make sure there's a
> certain number of dirty buffers (e.g. by doing CREATE TABLE AS
> some_query) and then do a manual checkpoint and time how long that
> times.
>
> Greetings,
>
> Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-11-04 08:25 ` Drouvot, Bertrand <[email protected]>
2022-11-15 11:41 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-15 20:18 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-12-07 19:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
0 siblings, 4 replies; 85+ messages in thread
From: Drouvot, Bertrand @ 2022-11-04 08:25 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; Andres Freund <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 7/28/22 11:38 AM, Nitin Jadhav wrote:
>>> To understand the performance effects of the above, I have taken the
>>> average of five checkpoints with the patch and without the patch in my
>>> environment. Here are the results.
>>> With patch: 269.65 s
>>> Without patch: 269.60 s
>>
>> Those look like timed checkpoints - if the checkpoints are sleeping a
>> part of the time, you're not going to see any potential overhead.
>
> Yes. The above data is collected from timed checkpoints.
>
> create table t1(a int);
> insert into t1 select * from generate_series(1,10000000);
>
> I generated a lot of data by using the above queries which would in
> turn trigger the checkpoint (wal).
> ---
>
>> To see whether this has an effect you'd have to make sure there's a
>> certain number of dirty buffers (e.g. by doing CREATE TABLE AS
>> some_query) and then do a manual checkpoint and time how long that
>> times.
>
> For this case I have generated data by using below queries.
>
> create table t1(a int);
> insert into t1 select * from generate_series(1,8000000);
>
> This does not trigger the checkpoint automatically. I have issued the
> CHECKPOINT manually and measured the performance by considering an
> average of 5 checkpoints. Here are the details.
>
> With patch: 2.457 s
> Without patch: 2.334 s
>
> Please share your thoughts.
>
v6 was not applying anymore, due to a change in
doc/src/sgml/ref/checkpoint.sgml done by b9eb0ff09e (Rename
pg_checkpointer predefined role to pg_checkpoint).
Please find attached a rebase in v7.
While working on this rebase, I also noticed that "pg_checkpointer" is
still mentioned in some translation files:
"
$ git grep pg_checkpointer
src/backend/po/de.po:msgid "must be superuser or have privileges of
pg_checkpointer to do CHECKPOINT"
src/backend/po/ja.po:msgid "must be superuser or have privileges of
pg_checkpointer to do CHECKPOINT"
src/backend/po/ja.po:msgstr
"CHECKPOINTを実行するにはスーパーユーザーであるか、またはpg_checkpointerの権限を持つ必要があります"
src/backend/po/sv.po:msgid "must be superuser or have privileges of
pg_checkpointer to do CHECKPOINT"
"
I'm not familiar with how the translation files are handled (looks like
they have their own set of commits, see 3c0bcdbc66 for example) but
wanted to mention that "pg_checkpointer" is still mentioned (even if
that may be expected as the last commit related to translation files
(aka 3c0bcdbc66) is older than the one that renamed pg_checkpointer to
pg_checkpoint (aka b9eb0ff09e)).
That said, back to this patch: I did not look closely but noticed that
the buffers_total reported by pg_stat_progress_checkpoint:
postgres=# select type,flags,start_lsn,phase,buffers_total,new_requests
from pg_stat_progress_checkpoint;
type | flags | start_lsn | phase
| buffers_total | new_requests
------------+-----------------------+------------+-----------------------+---------------+--------------
checkpoint | immediate force wait | 1/E6C523A8 | checkpointing
buffers | 1024275 | false
(1 row)
is a little bit different from what is logged once completed:
2022-11-04 08:18:50.806 UTC [3488442] LOG: checkpoint complete: wrote
1024278 buffers (97.7%);
Regards,
--
Bertrand Drouvot
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
commit 5198f5010be0febd019b1888817e2d78b8e42b21
Author: bdrouvot <[email protected]>
Date: Thu Nov 3 12:59:10 2022 +0000
v7-0001-pg_stat_progress_checkpoint-view.patch
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index e5d622d514..ceb7d60ffa 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -414,6 +414,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the checkpoint.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -5784,7 +5791,7 @@ FROM pg_stat_get_backend_idset() AS backendid;
which support progress reporting are <command>ANALYZE</command>,
<command>CLUSTER</command>,
<command>CREATE INDEX</command>, <command>VACUUM</command>,
- <command>COPY</command>,
+ <command>COPY</command>, <command>CHECKPOINT</command>
and <xref linkend="protocol-replication-base-backup"/> (i.e., replication
command that <xref linkend="app-pgbasebackup"/> issues to take
a base backup).
@@ -7072,6 +7079,402 @@ FROM pg_stat_get_backend_idset() AS backendid;
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>Checkpoint Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the checkpointer process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of the checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>flags</structfield> <type>text</type>
+ </para>
+ <para>
+ Flags of the checkpoint. See <xref linkend="checkpoint-flags"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_lsn</structfield> <type>text</type>
+ </para>
+ <para>
+ The checkpoint start location.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_time</structfield> <type>timestamp with time zone</type>
+ </para>
+ <para>
+ Start time of the checkpoint.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal>buffers_total</literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>buffers_total</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>files_total</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>new_requests</structfield> <type>text</type>
+ </para>
+ <para>
+ True if any of the backend is requested for a checkpoint while the
+ current checkpoint is in progress, False otherwise.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-types">
+ <title>Checkpoint Types</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Types</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>checkpoint</literal></entry>
+ <entry>
+ The current operation is checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>restartpoint</literal></entry>
+ <entry>
+ The current operation is restartpoint.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-flags">
+ <title>Checkpoint Flags</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Flags</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>shutdown</literal></entry>
+ <entry>
+ The checkpoint is for shutdown.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>end-of-recovery</literal></entry>
+ <entry>
+ The checkpoint is for end-of-recovery.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>immediate</literal></entry>
+ <entry>
+ The checkpoint happens without delays.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint is started because some operation (for which the
+ checkpoint is necessary) forced a checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>flush all</literal></entry>
+ <entry>
+ The checkpoint flushes all pages, including those belonging to unlogged
+ tables.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wait</literal></entry>
+ <entry>
+ The operations which requested the checkpoint waits for completion
+ before returning.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>requested</literal></entry>
+ <entry>
+ The checkpoint request has been made.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>max_wal_size</literal> is
+ reached.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>checkpoint_timeout</literal>
+ expired.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>Checkpoint Phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The checkpointer process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>getting virtual transaction IDs</literal></entry>
+ <entry>
+ The checkpointer process is getting the virtual transaction IDs that
+ are delaying the checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently flushing all the replication slots
+ to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical replication snapshot files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing all the serialized
+ snapshot files that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mapping files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing unwanted or flushing
+ required logical rewrite mapping files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication origin</literal></entry>
+ <entry>
+ The checkpointer process is currently performing a checkpoint of each
+ replication origin's progress with respect to the replayed remote LSN.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit log pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit log pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit time stamp pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit time stamp pages to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing subtransaction pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing subtransaction pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing multixact pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing multixact pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing predicate lock pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing predicate lock pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The checkpointer process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>processing file sync requests</literal></entry>
+ <entry>
+ The checkpointer process is currently processing file sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The checkpointer process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing post checkpoint cleanup</literal></entry>
+ <entry>
+ The checkpointer process is currently performing post checkpoint cleanup.
+ It removes any lingering files that can be safely removed.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>invalidating replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently invalidating replication slots.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old WAL files</literal></entry>
+ <entry>
+ The checkpointer process is currently recycling old WAL files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>truncating subtransactions</literal></entry>
+ <entry>
+ The checkpointer process is currently removing the subtransaction
+ segments.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The checkpointer process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/doc/src/sgml/ref/checkpoint.sgml b/doc/src/sgml/ref/checkpoint.sgml
index 28a1d717b8..906883336d 100644
--- a/doc/src/sgml/ref/checkpoint.sgml
+++ b/doc/src/sgml/ref/checkpoint.sgml
@@ -55,6 +55,14 @@ CHECKPOINT
Only superusers or users with the privileges of
the <link linkend="predefined-roles-table"><literal>pg_checkpoint</literal></link>
role can call <command>CHECKPOINT</command>.
+
+ <para>
+ The checkpointer process running the checkpoint will report its progress
+ in the <structname>pg_stat_progress_checkpoint</structname> view except for
+ the shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
+ </para>
+
</para>
</refsect1>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 6a38b53744..733a8d2837 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -531,7 +531,11 @@
adjust the <xref linkend="guc-archive-timeout"/> parameter rather than the
checkpoint parameters.)
It is also possible to force a checkpoint by using the SQL
- command <command>CHECKPOINT</command>.
+ command <command>CHECKPOINT</command>. The checkpointer process running the
+ checkpoint will report its progress in the
+ <structname>pg_stat_progress_checkpoint</structname> view except for the
+ shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
</para>
<para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index be54c23187..f2cebe0973 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -67,6 +67,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "commands/progress.h"
#include "common/controldata_utils.h"
#include "common/file_utils.h"
#include "executor/instrument.h"
@@ -698,6 +699,8 @@ static void WALInsertLockAcquireExclusive(void);
static void WALInsertLockRelease(void);
static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static void checkpoint_progress_start(int flags, int type);
+
/*
* Insert an XLOG record represented by an already-constructed chain of data
* chunks. This is a low-level routine; to construct the WAL record header
@@ -6622,6 +6625,9 @@ CreateCheckPoint(int flags)
XLogCtl->RedoRecPtr = checkPoint.redo;
SpinLockRelease(&XLogCtl->info_lck);
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_CHECKPOINT);
+
/*
* If enabled, log checkpoint start. We postpone this until now so as not
* to log anything if we decided to skip the checkpoint.
@@ -6704,6 +6710,8 @@ CreateCheckPoint(int flags)
* clog and we will correctly flush the update below. So we cannot miss
* any xacts we need to wait for.
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS);
vxids = GetVirtualXIDsDelayingChkpt(&nvxids, DELAY_CHKPT_START);
if (nvxids > 0)
{
@@ -6819,6 +6827,8 @@ CreateCheckPoint(int flags)
/*
* Let smgr do post-checkpoint cleanup (eg, deleting old files).
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP);
SyncPostCheckpoint();
/*
@@ -6834,6 +6844,9 @@ CreateCheckPoint(int flags)
*/
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
KeepLogSeg(recptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -6844,6 +6857,8 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
@@ -6862,11 +6877,21 @@ CreateCheckPoint(int flags)
* StartupSUBTRANS hasn't been called yet.
*/
if (!RecoveryInProgress())
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -7023,29 +7048,63 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN);
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES);
CheckPointPredicate();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SYNC_FILES);
ProcessSyncRequests();
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -7195,6 +7254,9 @@ CreateRestartPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the restartpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT);
+
if (log_checkpoints)
LogCheckpointStart(flags, true);
@@ -7278,6 +7340,9 @@ CreateRestartPoint(int flags)
replayPtr = GetXLogReplayRecPtr(&replayTLI);
endptr = (receivePtr < replayPtr) ? replayPtr : receivePtr;
KeepLogSeg(endptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -7304,6 +7369,8 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
/*
@@ -7320,11 +7387,20 @@ CreateRestartPoint(int flags)
* this because StartupSUBTRANS hasn't been called yet.
*/
if (EnableHotStandby)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the restartpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -8958,3 +9034,29 @@ SetWalWriterSleeping(bool sleeping)
XLogCtl->WalWriterSleeping = sleeping;
SpinLockRelease(&XLogCtl->info_lck);
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+static void
+checkpoint_progress_start(int flags, int type)
+{
+ const int index[] = {
+ PROGRESS_CHECKPOINT_TYPE,
+ PROGRESS_CHECKPOINT_FLAGS,
+ PROGRESS_CHECKPOINT_LSN,
+ PROGRESS_CHECKPOINT_START_TIMESTAMP,
+ PROGRESS_CHECKPOINT_PHASE
+ };
+ int64 val[5];
+
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+
+ val[0] = type;
+ val[1] = flags;
+ val[2] = RedoRecPtr;
+ val[3] = CheckpointStats.ckpt_start_t;
+ val[4] = PROGRESS_CHECKPOINT_PHASE_INIT;
+
+ pgstat_progress_update_multi_param(5, index, val);
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..384ca35833 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1267,6 +1267,57 @@ CREATE VIEW pg_stat_progress_copy AS
FROM pg_stat_get_progress_info('COPY') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ CASE S.param1 WHEN 1 THEN 'checkpoint'
+ WHEN 2 THEN 'restartpoint'
+ END AS type,
+ ( CASE WHEN (S.param2 & 4) > 0 THEN 'immediate ' ELSE '' END ||
+ CASE WHEN (S.param2 & 8) > 0 THEN 'force ' ELSE '' END ||
+ CASE WHEN (S.param2 & 16) > 0 THEN 'flush-all ' ELSE '' END ||
+ CASE WHEN (S.param2 & 32) > 0 THEN 'wait ' ELSE '' END ||
+ CASE WHEN (S.param2 & 128) > 0 THEN 'wal ' ELSE '' END ||
+ CASE WHEN (S.param2 & 256) > 0 THEN 'time ' ELSE '' END
+ ) AS flags,
+ ( '0/0'::pg_lsn +
+ ((CASE
+ WHEN S.param3 < 0 THEN pow(2::numeric, 64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ S.param3::numeric)
+ ) AS start_lsn,
+ to_timestamp(946684800 + (S.param4::float8 / 1000000)) AS start_time,
+ CASE S.param5 WHEN 1 THEN 'initializing'
+ WHEN 2 THEN 'getting virtual transaction IDs'
+ WHEN 3 THEN 'checkpointing replication slots'
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'
+ WHEN 6 THEN 'checkpointing replication origin'
+ WHEN 7 THEN 'checkpointing commit log pages'
+ WHEN 8 THEN 'checkpointing commit time stamp pages'
+ WHEN 9 THEN 'checkpointing subtransaction pages'
+ WHEN 10 THEN 'checkpointing multixact pages'
+ WHEN 11 THEN 'checkpointing predicate lock pages'
+ WHEN 12 THEN 'checkpointing buffers'
+ WHEN 13 THEN 'processing file sync requests'
+ WHEN 14 THEN 'performing two phase checkpoint'
+ WHEN 15 THEN 'performing post checkpoint cleanup'
+ WHEN 16 THEN 'invalidating replication slots'
+ WHEN 17 THEN 'recycling old WAL files'
+ WHEN 18 THEN 'truncating subtransactions'
+ WHEN 19 THEN 'finalizing'
+ END AS phase,
+ S.param6 AS buffers_total,
+ S.param7 AS buffers_processed,
+ S.param8 AS buffers_written,
+ S.param9 AS files_total,
+ S.param10 AS files_synced,
+ CASE S.param11 WHEN 0 THEN 'false'
+ WHEN 1 THEN 'true'
+ END AS new_requests
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
+
CREATE VIEW pg_user_mappings AS
SELECT
U.oid AS umid,
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 5fc076fc14..21bf75b058 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -39,6 +39,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogrecovery.h"
+#include "commands/progress.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -163,7 +164,7 @@ static pg_time_t last_xlog_switch_time;
static void HandleCheckpointerInterrupts(void);
static void CheckArchiveTimeout(void);
static bool IsCheckpointOnSchedule(double progress);
-static bool ImmediateCheckpointRequested(void);
+static bool ImmediateCheckpointRequested(int flags);
static bool CompactCheckpointerRequestQueue(void);
static void UpdateSharedMemoryConfig(void);
@@ -667,16 +668,24 @@ CheckArchiveTimeout(void)
* there is one pending behind it.)
*/
static bool
-ImmediateCheckpointRequested(void)
+ImmediateCheckpointRequested(int flags)
{
volatile CheckpointerShmemStruct *cps = CheckpointerShmem;
+ if (cps->ckpt_flags & CHECKPOINT_REQUESTED)
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_NEW_REQUESTS, true);
+
/*
* We don't need to acquire the ckpt_lck in this case because we're only
* looking at a single flag bit.
*/
if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FLAGS,
+ (flags | CHECKPOINT_IMMEDIATE));
return true;
+ }
+
return false;
}
@@ -708,7 +717,7 @@ CheckpointWriteDelay(int flags, double progress)
*/
if (!(flags & CHECKPOINT_IMMEDIATE) &&
!ShutdownRequestPending &&
- !ImmediateCheckpointRequested() &&
+ !ImmediateCheckpointRequested(flags) &&
IsCheckpointOnSchedule(progress))
{
if (ConfigReloadPending)
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 73d30bf619..6d69255667 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -39,6 +39,7 @@
#include "catalog/catalog.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2026,6 +2027,8 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_TOTAL,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2143,6 +2146,8 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2163,6 +2168,8 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.buf_written_checkpoints++;
num_written++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index 9d6a9e9109..aa25215910 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -368,6 +369,9 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_TOTAL,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -431,6 +435,8 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 96bffc0f2a..8281d961bf 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -497,6 +497,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..33a64d2f0b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,42 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_TYPE 0
+#define PROGRESS_CHECKPOINT_FLAGS 1
+#define PROGRESS_CHECKPOINT_LSN 2
+#define PROGRESS_CHECKPOINT_START_TIMESTAMP 3
+#define PROGRESS_CHECKPOINT_PHASE 4
+#define PROGRESS_CHECKPOINT_BUFFERS_TOTAL 5
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 6
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 7
+#define PROGRESS_CHECKPOINT_FILES_TOTAL 8
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 9
+#define PROGRESS_CHECKPOINT_NEW_REQUESTS 10
+
+/* Types of checkpoint (as advertised via PROGRESS_CHECKPOINT_TYPE) */
+#define PROGRESS_CHECKPOINT_TYPE_CHECKPOINT 1
+#define PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT 2
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 1
+#define PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS 2
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 3
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 4
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 5
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN 6
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 9
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 10
+#define PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES 11
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 12
+#define PROGRESS_CHECKPOINT_PHASE_SYNC_FILES 13
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 14
+#define PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP 15
+#define PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS 16
+#define PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG 17
+#define PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS 18
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 19
+
#endif
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 624d0e5aae..eab68d7f14 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1913,6 +1913,76 @@ pg_stat_progress_basebackup| SELECT s.pid,
s.param4 AS tablespaces_total,
s.param5 AS tablespaces_streamed
FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
+pg_stat_progress_checkpoint| SELECT s.pid,
+ CASE s.param1
+ WHEN 1 THEN 'checkpoint'::text
+ WHEN 2 THEN 'restartpoint'::text
+ ELSE NULL::text
+ END AS type,
+ (((((
+ CASE
+ WHEN ((s.param2 & (4)::bigint) > 0) THEN 'immediate '::text
+ ELSE ''::text
+ END ||
+ CASE
+ WHEN ((s.param2 & (8)::bigint) > 0) THEN 'force '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (16)::bigint) > 0) THEN 'flush-all '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (32)::bigint) > 0) THEN 'wait '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (128)::bigint) > 0) THEN 'wal '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (256)::bigint) > 0) THEN 'time '::text
+ ELSE ''::text
+ END) AS flags,
+ ('0/0'::pg_lsn + (
+ CASE
+ WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (s.param3)::numeric)) AS start_lsn,
+ to_timestamp(((946684800)::double precision + ((s.param4)::double precision / (1000000)::double precision))) AS start_time,
+ CASE s.param5
+ WHEN 1 THEN 'initializing'::text
+ WHEN 2 THEN 'getting virtual transaction IDs'::text
+ WHEN 3 THEN 'checkpointing replication slots'::text
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'::text
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'::text
+ WHEN 6 THEN 'checkpointing replication origin'::text
+ WHEN 7 THEN 'checkpointing commit log pages'::text
+ WHEN 8 THEN 'checkpointing commit time stamp pages'::text
+ WHEN 9 THEN 'checkpointing subtransaction pages'::text
+ WHEN 10 THEN 'checkpointing multixact pages'::text
+ WHEN 11 THEN 'checkpointing predicate lock pages'::text
+ WHEN 12 THEN 'checkpointing buffers'::text
+ WHEN 13 THEN 'processing file sync requests'::text
+ WHEN 14 THEN 'performing two phase checkpoint'::text
+ WHEN 15 THEN 'performing post checkpoint cleanup'::text
+ WHEN 16 THEN 'invalidating replication slots'::text
+ WHEN 17 THEN 'recycling old WAL files'::text
+ WHEN 18 THEN 'truncating subtransactions'::text
+ WHEN 19 THEN 'finalizing'::text
+ ELSE NULL::text
+ END AS phase,
+ s.param6 AS buffers_total,
+ s.param7 AS buffers_processed,
+ s.param8 AS buffers_written,
+ s.param9 AS files_total,
+ s.param10 AS files_synced,
+ CASE s.param11
+ WHEN 0 THEN 'false'::text
+ WHEN 1 THEN 'true'::text
+ ELSE NULL::text
+ END AS new_requests
+ FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
pg_stat_progress_cluster| SELECT s.pid,
s.datid,
d.datname,
Attachments:
[text/plain] v7-0001-pg_stat_progress_checkpoint-view.patch (37.3K, ../../[email protected]/2-v7-0001-pg_stat_progress_checkpoint-view.patch)
download | inline diff:
commit 5198f5010be0febd019b1888817e2d78b8e42b21
Author: bdrouvot <[email protected]>
Date: Thu Nov 3 12:59:10 2022 +0000
v7-0001-pg_stat_progress_checkpoint-view.patch
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index e5d622d514..ceb7d60ffa 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -414,6 +414,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the checkpoint.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -5784,7 +5791,7 @@ FROM pg_stat_get_backend_idset() AS backendid;
which support progress reporting are <command>ANALYZE</command>,
<command>CLUSTER</command>,
<command>CREATE INDEX</command>, <command>VACUUM</command>,
- <command>COPY</command>,
+ <command>COPY</command>, <command>CHECKPOINT</command>
and <xref linkend="protocol-replication-base-backup"/> (i.e., replication
command that <xref linkend="app-pgbasebackup"/> issues to take
a base backup).
@@ -7072,6 +7079,402 @@ FROM pg_stat_get_backend_idset() AS backendid;
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>Checkpoint Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the checkpointer process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of the checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>flags</structfield> <type>text</type>
+ </para>
+ <para>
+ Flags of the checkpoint. See <xref linkend="checkpoint-flags"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_lsn</structfield> <type>text</type>
+ </para>
+ <para>
+ The checkpoint start location.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_time</structfield> <type>timestamp with time zone</type>
+ </para>
+ <para>
+ Start time of the checkpoint.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal>buffers_total</literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>buffers_total</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>files_total</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>new_requests</structfield> <type>text</type>
+ </para>
+ <para>
+ True if any of the backend is requested for a checkpoint while the
+ current checkpoint is in progress, False otherwise.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-types">
+ <title>Checkpoint Types</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Types</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>checkpoint</literal></entry>
+ <entry>
+ The current operation is checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>restartpoint</literal></entry>
+ <entry>
+ The current operation is restartpoint.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-flags">
+ <title>Checkpoint Flags</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Flags</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>shutdown</literal></entry>
+ <entry>
+ The checkpoint is for shutdown.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>end-of-recovery</literal></entry>
+ <entry>
+ The checkpoint is for end-of-recovery.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>immediate</literal></entry>
+ <entry>
+ The checkpoint happens without delays.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint is started because some operation (for which the
+ checkpoint is necessary) forced a checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>flush all</literal></entry>
+ <entry>
+ The checkpoint flushes all pages, including those belonging to unlogged
+ tables.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wait</literal></entry>
+ <entry>
+ The operations which requested the checkpoint waits for completion
+ before returning.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>requested</literal></entry>
+ <entry>
+ The checkpoint request has been made.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>max_wal_size</literal> is
+ reached.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>checkpoint_timeout</literal>
+ expired.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>Checkpoint Phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The checkpointer process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>getting virtual transaction IDs</literal></entry>
+ <entry>
+ The checkpointer process is getting the virtual transaction IDs that
+ are delaying the checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently flushing all the replication slots
+ to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical replication snapshot files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing all the serialized
+ snapshot files that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mapping files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing unwanted or flushing
+ required logical rewrite mapping files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication origin</literal></entry>
+ <entry>
+ The checkpointer process is currently performing a checkpoint of each
+ replication origin's progress with respect to the replayed remote LSN.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit log pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit log pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit time stamp pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit time stamp pages to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing subtransaction pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing subtransaction pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing multixact pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing multixact pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing predicate lock pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing predicate lock pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The checkpointer process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>processing file sync requests</literal></entry>
+ <entry>
+ The checkpointer process is currently processing file sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The checkpointer process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing post checkpoint cleanup</literal></entry>
+ <entry>
+ The checkpointer process is currently performing post checkpoint cleanup.
+ It removes any lingering files that can be safely removed.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>invalidating replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently invalidating replication slots.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old WAL files</literal></entry>
+ <entry>
+ The checkpointer process is currently recycling old WAL files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>truncating subtransactions</literal></entry>
+ <entry>
+ The checkpointer process is currently removing the subtransaction
+ segments.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The checkpointer process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/doc/src/sgml/ref/checkpoint.sgml b/doc/src/sgml/ref/checkpoint.sgml
index 28a1d717b8..906883336d 100644
--- a/doc/src/sgml/ref/checkpoint.sgml
+++ b/doc/src/sgml/ref/checkpoint.sgml
@@ -55,6 +55,14 @@ CHECKPOINT
Only superusers or users with the privileges of
the <link linkend="predefined-roles-table"><literal>pg_checkpoint</literal></link>
role can call <command>CHECKPOINT</command>.
+
+ <para>
+ The checkpointer process running the checkpoint will report its progress
+ in the <structname>pg_stat_progress_checkpoint</structname> view except for
+ the shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
+ </para>
+
</para>
</refsect1>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 6a38b53744..733a8d2837 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -531,7 +531,11 @@
adjust the <xref linkend="guc-archive-timeout"/> parameter rather than the
checkpoint parameters.)
It is also possible to force a checkpoint by using the SQL
- command <command>CHECKPOINT</command>.
+ command <command>CHECKPOINT</command>. The checkpointer process running the
+ checkpoint will report its progress in the
+ <structname>pg_stat_progress_checkpoint</structname> view except for the
+ shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
</para>
<para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index be54c23187..f2cebe0973 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -67,6 +67,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "commands/progress.h"
#include "common/controldata_utils.h"
#include "common/file_utils.h"
#include "executor/instrument.h"
@@ -698,6 +699,8 @@ static void WALInsertLockAcquireExclusive(void);
static void WALInsertLockRelease(void);
static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static void checkpoint_progress_start(int flags, int type);
+
/*
* Insert an XLOG record represented by an already-constructed chain of data
* chunks. This is a low-level routine; to construct the WAL record header
@@ -6622,6 +6625,9 @@ CreateCheckPoint(int flags)
XLogCtl->RedoRecPtr = checkPoint.redo;
SpinLockRelease(&XLogCtl->info_lck);
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_CHECKPOINT);
+
/*
* If enabled, log checkpoint start. We postpone this until now so as not
* to log anything if we decided to skip the checkpoint.
@@ -6704,6 +6710,8 @@ CreateCheckPoint(int flags)
* clog and we will correctly flush the update below. So we cannot miss
* any xacts we need to wait for.
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS);
vxids = GetVirtualXIDsDelayingChkpt(&nvxids, DELAY_CHKPT_START);
if (nvxids > 0)
{
@@ -6819,6 +6827,8 @@ CreateCheckPoint(int flags)
/*
* Let smgr do post-checkpoint cleanup (eg, deleting old files).
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP);
SyncPostCheckpoint();
/*
@@ -6834,6 +6844,9 @@ CreateCheckPoint(int flags)
*/
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
KeepLogSeg(recptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -6844,6 +6857,8 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
@@ -6862,11 +6877,21 @@ CreateCheckPoint(int flags)
* StartupSUBTRANS hasn't been called yet.
*/
if (!RecoveryInProgress())
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -7023,29 +7048,63 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN);
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES);
CheckPointPredicate();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SYNC_FILES);
ProcessSyncRequests();
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -7195,6 +7254,9 @@ CreateRestartPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the restartpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT);
+
if (log_checkpoints)
LogCheckpointStart(flags, true);
@@ -7278,6 +7340,9 @@ CreateRestartPoint(int flags)
replayPtr = GetXLogReplayRecPtr(&replayTLI);
endptr = (receivePtr < replayPtr) ? replayPtr : receivePtr;
KeepLogSeg(endptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -7304,6 +7369,8 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
/*
@@ -7320,11 +7387,20 @@ CreateRestartPoint(int flags)
* this because StartupSUBTRANS hasn't been called yet.
*/
if (EnableHotStandby)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the restartpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -8958,3 +9034,29 @@ SetWalWriterSleeping(bool sleeping)
XLogCtl->WalWriterSleeping = sleeping;
SpinLockRelease(&XLogCtl->info_lck);
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+static void
+checkpoint_progress_start(int flags, int type)
+{
+ const int index[] = {
+ PROGRESS_CHECKPOINT_TYPE,
+ PROGRESS_CHECKPOINT_FLAGS,
+ PROGRESS_CHECKPOINT_LSN,
+ PROGRESS_CHECKPOINT_START_TIMESTAMP,
+ PROGRESS_CHECKPOINT_PHASE
+ };
+ int64 val[5];
+
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+
+ val[0] = type;
+ val[1] = flags;
+ val[2] = RedoRecPtr;
+ val[3] = CheckpointStats.ckpt_start_t;
+ val[4] = PROGRESS_CHECKPOINT_PHASE_INIT;
+
+ pgstat_progress_update_multi_param(5, index, val);
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2d8104b090..384ca35833 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1267,6 +1267,57 @@ CREATE VIEW pg_stat_progress_copy AS
FROM pg_stat_get_progress_info('COPY') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ CASE S.param1 WHEN 1 THEN 'checkpoint'
+ WHEN 2 THEN 'restartpoint'
+ END AS type,
+ ( CASE WHEN (S.param2 & 4) > 0 THEN 'immediate ' ELSE '' END ||
+ CASE WHEN (S.param2 & 8) > 0 THEN 'force ' ELSE '' END ||
+ CASE WHEN (S.param2 & 16) > 0 THEN 'flush-all ' ELSE '' END ||
+ CASE WHEN (S.param2 & 32) > 0 THEN 'wait ' ELSE '' END ||
+ CASE WHEN (S.param2 & 128) > 0 THEN 'wal ' ELSE '' END ||
+ CASE WHEN (S.param2 & 256) > 0 THEN 'time ' ELSE '' END
+ ) AS flags,
+ ( '0/0'::pg_lsn +
+ ((CASE
+ WHEN S.param3 < 0 THEN pow(2::numeric, 64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ S.param3::numeric)
+ ) AS start_lsn,
+ to_timestamp(946684800 + (S.param4::float8 / 1000000)) AS start_time,
+ CASE S.param5 WHEN 1 THEN 'initializing'
+ WHEN 2 THEN 'getting virtual transaction IDs'
+ WHEN 3 THEN 'checkpointing replication slots'
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'
+ WHEN 6 THEN 'checkpointing replication origin'
+ WHEN 7 THEN 'checkpointing commit log pages'
+ WHEN 8 THEN 'checkpointing commit time stamp pages'
+ WHEN 9 THEN 'checkpointing subtransaction pages'
+ WHEN 10 THEN 'checkpointing multixact pages'
+ WHEN 11 THEN 'checkpointing predicate lock pages'
+ WHEN 12 THEN 'checkpointing buffers'
+ WHEN 13 THEN 'processing file sync requests'
+ WHEN 14 THEN 'performing two phase checkpoint'
+ WHEN 15 THEN 'performing post checkpoint cleanup'
+ WHEN 16 THEN 'invalidating replication slots'
+ WHEN 17 THEN 'recycling old WAL files'
+ WHEN 18 THEN 'truncating subtransactions'
+ WHEN 19 THEN 'finalizing'
+ END AS phase,
+ S.param6 AS buffers_total,
+ S.param7 AS buffers_processed,
+ S.param8 AS buffers_written,
+ S.param9 AS files_total,
+ S.param10 AS files_synced,
+ CASE S.param11 WHEN 0 THEN 'false'
+ WHEN 1 THEN 'true'
+ END AS new_requests
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
+
CREATE VIEW pg_user_mappings AS
SELECT
U.oid AS umid,
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index 5fc076fc14..21bf75b058 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -39,6 +39,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogrecovery.h"
+#include "commands/progress.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -163,7 +164,7 @@ static pg_time_t last_xlog_switch_time;
static void HandleCheckpointerInterrupts(void);
static void CheckArchiveTimeout(void);
static bool IsCheckpointOnSchedule(double progress);
-static bool ImmediateCheckpointRequested(void);
+static bool ImmediateCheckpointRequested(int flags);
static bool CompactCheckpointerRequestQueue(void);
static void UpdateSharedMemoryConfig(void);
@@ -667,16 +668,24 @@ CheckArchiveTimeout(void)
* there is one pending behind it.)
*/
static bool
-ImmediateCheckpointRequested(void)
+ImmediateCheckpointRequested(int flags)
{
volatile CheckpointerShmemStruct *cps = CheckpointerShmem;
+ if (cps->ckpt_flags & CHECKPOINT_REQUESTED)
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_NEW_REQUESTS, true);
+
/*
* We don't need to acquire the ckpt_lck in this case because we're only
* looking at a single flag bit.
*/
if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FLAGS,
+ (flags | CHECKPOINT_IMMEDIATE));
return true;
+ }
+
return false;
}
@@ -708,7 +717,7 @@ CheckpointWriteDelay(int flags, double progress)
*/
if (!(flags & CHECKPOINT_IMMEDIATE) &&
!ShutdownRequestPending &&
- !ImmediateCheckpointRequested() &&
+ !ImmediateCheckpointRequested(flags) &&
IsCheckpointOnSchedule(progress))
{
if (ConfigReloadPending)
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index 73d30bf619..6d69255667 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -39,6 +39,7 @@
#include "catalog/catalog.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2026,6 +2027,8 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_TOTAL,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2143,6 +2146,8 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2163,6 +2168,8 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.buf_written_checkpoints++;
num_written++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index 9d6a9e9109..aa25215910 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -368,6 +369,9 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_TOTAL,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -431,6 +435,8 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 96bffc0f2a..8281d961bf 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -497,6 +497,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..33a64d2f0b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,42 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_TYPE 0
+#define PROGRESS_CHECKPOINT_FLAGS 1
+#define PROGRESS_CHECKPOINT_LSN 2
+#define PROGRESS_CHECKPOINT_START_TIMESTAMP 3
+#define PROGRESS_CHECKPOINT_PHASE 4
+#define PROGRESS_CHECKPOINT_BUFFERS_TOTAL 5
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 6
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 7
+#define PROGRESS_CHECKPOINT_FILES_TOTAL 8
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 9
+#define PROGRESS_CHECKPOINT_NEW_REQUESTS 10
+
+/* Types of checkpoint (as advertised via PROGRESS_CHECKPOINT_TYPE) */
+#define PROGRESS_CHECKPOINT_TYPE_CHECKPOINT 1
+#define PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT 2
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 1
+#define PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS 2
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 3
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 4
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 5
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN 6
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 9
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 10
+#define PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES 11
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 12
+#define PROGRESS_CHECKPOINT_PHASE_SYNC_FILES 13
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 14
+#define PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP 15
+#define PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS 16
+#define PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG 17
+#define PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS 18
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 19
+
#endif
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 624d0e5aae..eab68d7f14 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1913,6 +1913,76 @@ pg_stat_progress_basebackup| SELECT s.pid,
s.param4 AS tablespaces_total,
s.param5 AS tablespaces_streamed
FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
+pg_stat_progress_checkpoint| SELECT s.pid,
+ CASE s.param1
+ WHEN 1 THEN 'checkpoint'::text
+ WHEN 2 THEN 'restartpoint'::text
+ ELSE NULL::text
+ END AS type,
+ (((((
+ CASE
+ WHEN ((s.param2 & (4)::bigint) > 0) THEN 'immediate '::text
+ ELSE ''::text
+ END ||
+ CASE
+ WHEN ((s.param2 & (8)::bigint) > 0) THEN 'force '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (16)::bigint) > 0) THEN 'flush-all '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (32)::bigint) > 0) THEN 'wait '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (128)::bigint) > 0) THEN 'wal '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (256)::bigint) > 0) THEN 'time '::text
+ ELSE ''::text
+ END) AS flags,
+ ('0/0'::pg_lsn + (
+ CASE
+ WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (s.param3)::numeric)) AS start_lsn,
+ to_timestamp(((946684800)::double precision + ((s.param4)::double precision / (1000000)::double precision))) AS start_time,
+ CASE s.param5
+ WHEN 1 THEN 'initializing'::text
+ WHEN 2 THEN 'getting virtual transaction IDs'::text
+ WHEN 3 THEN 'checkpointing replication slots'::text
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'::text
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'::text
+ WHEN 6 THEN 'checkpointing replication origin'::text
+ WHEN 7 THEN 'checkpointing commit log pages'::text
+ WHEN 8 THEN 'checkpointing commit time stamp pages'::text
+ WHEN 9 THEN 'checkpointing subtransaction pages'::text
+ WHEN 10 THEN 'checkpointing multixact pages'::text
+ WHEN 11 THEN 'checkpointing predicate lock pages'::text
+ WHEN 12 THEN 'checkpointing buffers'::text
+ WHEN 13 THEN 'processing file sync requests'::text
+ WHEN 14 THEN 'performing two phase checkpoint'::text
+ WHEN 15 THEN 'performing post checkpoint cleanup'::text
+ WHEN 16 THEN 'invalidating replication slots'::text
+ WHEN 17 THEN 'recycling old WAL files'::text
+ WHEN 18 THEN 'truncating subtransactions'::text
+ WHEN 19 THEN 'finalizing'::text
+ ELSE NULL::text
+ END AS phase,
+ s.param6 AS buffers_total,
+ s.param7 AS buffers_processed,
+ s.param8 AS buffers_written,
+ s.param9 AS files_total,
+ s.param10 AS files_synced,
+ CASE s.param11
+ WHEN 0 THEN 'false'::text
+ WHEN 1 THEN 'true'::text
+ ELSE NULL::text
+ END AS new_requests
+ FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
pg_stat_progress_cluster| SELECT s.pid,
s.datid,
d.datname,
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
@ 2022-11-15 11:41 ` Nitin Jadhav <[email protected]>
3 siblings, 0 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-11-15 11:41 UTC (permalink / raw)
To: Drouvot, Bertrand <[email protected]>; +Cc: Andres Freund <[email protected]>; Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> v6 was not applying anymore, due to a change in
> doc/src/sgml/ref/checkpoint.sgml done by b9eb0ff09e (Rename
> pg_checkpointer predefined role to pg_checkpoint).
>
> Please find attached a rebase in v7.
>
> While working on this rebase, I also noticed that "pg_checkpointer" is
> still mentioned in some translation files:
Thanks for rebasing the patch and sharing the information.
---
> That said, back to this patch: I did not look closely but noticed that
> the buffers_total reported by pg_stat_progress_checkpoint:
>
> postgres=# select type,flags,start_lsn,phase,buffers_total,new_requests
> from pg_stat_progress_checkpoint;
> type | flags | start_lsn | phase
> | buffers_total | new_requests
> ------------+-----------------------+------------+-----------------------+---------------+--------------
> checkpoint | immediate force wait | 1/E6C523A8 | checkpointing
> buffers | 1024275 | false
> (1 row)
>
> is a little bit different from what is logged once completed:
>
> 2022-11-04 08:18:50.806 UTC [3488442] LOG: checkpoint complete: wrote
> 1024278 buffers (97.7%);
This is because the count shown in the checkpoint complete message
includes the additional increment done during SlruInternalWritePage().
We are not sure of this increment until it really happens. Hence it
was not considered in the patch. To make it compatible with the
checkpoint complete message, we should increment all three here,
buffers_total, buffers_processed and buffers_written. So the total
number of buffers calculated earlier may not always be the same. If
this looks good, I will update this in the next patch.
Thanks & Regards,
Nitin Jadhav
On Fri, Nov 4, 2022 at 1:57 PM Drouvot, Bertrand
<[email protected]> wrote:
>
> Hi,
>
> On 7/28/22 11:38 AM, Nitin Jadhav wrote:
> >>> To understand the performance effects of the above, I have taken the
> >>> average of five checkpoints with the patch and without the patch in my
> >>> environment. Here are the results.
> >>> With patch: 269.65 s
> >>> Without patch: 269.60 s
> >>
> >> Those look like timed checkpoints - if the checkpoints are sleeping a
> >> part of the time, you're not going to see any potential overhead.
> >
> > Yes. The above data is collected from timed checkpoints.
> >
> > create table t1(a int);
> > insert into t1 select * from generate_series(1,10000000);
> >
> > I generated a lot of data by using the above queries which would in
> > turn trigger the checkpoint (wal).
> > ---
> >
> >> To see whether this has an effect you'd have to make sure there's a
> >> certain number of dirty buffers (e.g. by doing CREATE TABLE AS
> >> some_query) and then do a manual checkpoint and time how long that
> >> times.
> >
> > For this case I have generated data by using below queries.
> >
> > create table t1(a int);
> > insert into t1 select * from generate_series(1,8000000);
> >
> > This does not trigger the checkpoint automatically. I have issued the
> > CHECKPOINT manually and measured the performance by considering an
> > average of 5 checkpoints. Here are the details.
> >
> > With patch: 2.457 s
> > Without patch: 2.334 s
> >
> > Please share your thoughts.
> >
>
> v6 was not applying anymore, due to a change in
> doc/src/sgml/ref/checkpoint.sgml done by b9eb0ff09e (Rename
> pg_checkpointer predefined role to pg_checkpoint).
>
> Please find attached a rebase in v7.
>
> While working on this rebase, I also noticed that "pg_checkpointer" is
> still mentioned in some translation files:
> "
> $ git grep pg_checkpointer
> src/backend/po/de.po:msgid "must be superuser or have privileges of
> pg_checkpointer to do CHECKPOINT"
> src/backend/po/ja.po:msgid "must be superuser or have privileges of
> pg_checkpointer to do CHECKPOINT"
> src/backend/po/ja.po:msgstr
> "CHECKPOINTを実行するにはスーパーユーザーであるか、またはpg_checkpointerの権限を持つ必要があります"
> src/backend/po/sv.po:msgid "must be superuser or have privileges of
> pg_checkpointer to do CHECKPOINT"
> "
>
> I'm not familiar with how the translation files are handled (looks like
> they have their own set of commits, see 3c0bcdbc66 for example) but
> wanted to mention that "pg_checkpointer" is still mentioned (even if
> that may be expected as the last commit related to translation files
> (aka 3c0bcdbc66) is older than the one that renamed pg_checkpointer to
> pg_checkpoint (aka b9eb0ff09e)).
>
> That said, back to this patch: I did not look closely but noticed that
> the buffers_total reported by pg_stat_progress_checkpoint:
>
> postgres=# select type,flags,start_lsn,phase,buffers_total,new_requests
> from pg_stat_progress_checkpoint;
> type | flags | start_lsn | phase
> | buffers_total | new_requests
> ------------+-----------------------+------------+-----------------------+---------------+--------------
> checkpoint | immediate force wait | 1/E6C523A8 | checkpointing
> buffers | 1024275 | false
> (1 row)
>
> is a little bit different from what is logged once completed:
>
> 2022-11-04 08:18:50.806 UTC [3488442] LOG: checkpoint complete: wrote
> 1024278 buffers (97.7%);
>
> Regards,
>
> --
> Bertrand Drouvot
> PostgreSQL Contributors Team
> RDS Open Source Databases
> Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
@ 2022-11-15 20:04 ` Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
3 siblings, 1 reply; 85+ messages in thread
From: Robert Haas @ 2022-11-15 20:04 UTC (permalink / raw)
To: Drouvot, Bertrand <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Andres Freund <[email protected]>; Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Nov 4, 2022 at 4:27 AM Drouvot, Bertrand
<[email protected]> wrote:
> Please find attached a rebase in v7.
I don't think it's a good thing that this patch is using the
progress-reporting machinery. The point of that machinery is that we
want any backend to be able to report progress for any command it
happens to be running, and we don't know which command that will be at
any given point in time, or how many backends will be running any
given command at once. So we need some generic set of counters that
can be repurposed for whatever any particular backend happens to be
doing right at the moment.
But none of that applies to the checkpointer. Any information about
the checkpointer that we want to expose can just be advertised in a
dedicated chunk of shared memory, perhaps even by simply adding it to
CheckpointerShmemStruct. Then you can give the fields whatever names,
types, and sizes you like, and you don't have to do all of this stuff
with mapping down to integers and back. The only real disadvantage
that I can see is then you have to think a bit harder about what the
concurrency model is here, and maybe you end up reimplementing
something similar to what the progress-reporting stuff does for you,
and *maybe* that is a sufficient reason to do it this way.
But I'm doubtful. This feels like a square-peg-round-hole situation.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
@ 2022-11-16 10:31 ` Bharath Rupireddy <[email protected]>
2022-11-16 18:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
0 siblings, 2 replies; 85+ messages in thread
From: Bharath Rupireddy @ 2022-11-16 10:31 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Andres Freund <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Nov 16, 2022 at 1:35 AM Robert Haas <[email protected]> wrote:
>
> On Fri, Nov 4, 2022 at 4:27 AM Drouvot, Bertrand
> <[email protected]> wrote:
> > Please find attached a rebase in v7.
>
> I don't think it's a good thing that this patch is using the
> progress-reporting machinery. The point of that machinery is that we
> want any backend to be able to report progress for any command it
> happens to be running, and we don't know which command that will be at
> any given point in time, or how many backends will be running any
> given command at once. So we need some generic set of counters that
> can be repurposed for whatever any particular backend happens to be
> doing right at the moment.
Hm.
> But none of that applies to the checkpointer. Any information about
> the checkpointer that we want to expose can just be advertised in a
> dedicated chunk of shared memory, perhaps even by simply adding it to
> CheckpointerShmemStruct. Then you can give the fields whatever names,
> types, and sizes you like, and you don't have to do all of this stuff
> with mapping down to integers and back. The only real disadvantage
> that I can see is then you have to think a bit harder about what the
> concurrency model is here, and maybe you end up reimplementing
> something similar to what the progress-reporting stuff does for you,
> and *maybe* that is a sufficient reason to do it this way.
-1 for CheckpointerShmemStruct as it is being used for running
checkpoints and I don't think adding stats to it is a great idea.
Instead, extending PgStat_CheckpointerStats and using shared memory
stats for reporting progress/last checkpoint related stats is a good
idea IMO. I also think that a new pg_stat_checkpoint view is needed
because, right now, the PgStat_CheckpointerStats stats are exposed via
the pg_stat_bgwriter view, having a separate view for checkpoint stats
is good here. Also, removing CheckpointStatsData and moving all of
those members to PgStat_CheckpointerStats, of course, by being careful
about the amount of shared memory required, is also a good idea IMO.
Going forward, PgStat_CheckpointerStats and pg_stat_checkpoint view
can be a single point of location for all the checkpoint related
stats.
Thoughts?
In fact, I was recently having an off-list chat with Bertrand Drouvot
about the above idea.
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
@ 2022-11-16 18:14 ` Andres Freund <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Andres Freund @ 2022-11-16 18:14 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Robert Haas <[email protected]>; Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 2022-11-16 16:01:55 +0530, Bharath Rupireddy wrote:
> -1 for CheckpointerShmemStruct as it is being used for running
> checkpoints and I don't think adding stats to it is a great idea.
Why? Imo the data needed for progress reporting aren't really "stats". We'd
not accumulate counters over time, just for the current checkpoint.
I think it might even be useful for other parts of the system to know what the
checkpointer is doing, e.g. bgwriter or autovacuum could adapt the behaviour
if checkpointer can't keep up. Somehow it'd feel wrong to use the stats system
as the source of such adjustments - but perhaps my gut feeling on that isn't
right.
The best argument for combining progress reporting with accumulating stats is
that we could likely share some of the code. Having accumulated stats for all
the checkpoint phases would e.g. be quite valuable.
> Instead, extending PgStat_CheckpointerStats and using shared memory
> stats for reporting progress/last checkpoint related stats is a good
> idea IMO
There's certainly some potential for deduplicating state and to make stats
updated more frequently. But that doesn't necessarily mean that putting the
checkpoint progress into PgStat_CheckpointerStats is a good idea (nor the
opposite).
> I also think that a new pg_stat_checkpoint view is needed
> because, right now, the PgStat_CheckpointerStats stats are exposed via
> the pg_stat_bgwriter view, having a separate view for checkpoint stats
> is good here.
I agree that we should do that, but largely independent of the architectural
question at hand.
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
@ 2022-11-16 19:19 ` Robert Haas <[email protected]>
2022-11-16 19:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-17 13:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
1 sibling, 2 replies; 85+ messages in thread
From: Robert Haas @ 2022-11-16 19:19 UTC (permalink / raw)
To: Bharath Rupireddy <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Andres Freund <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Nov 16, 2022 at 5:32 AM Bharath Rupireddy
<[email protected]> wrote:
> -1 for CheckpointerShmemStruct as it is being used for running
> checkpoints and I don't think adding stats to it is a great idea.
> Instead, extending PgStat_CheckpointerStats and using shared memory
> stats for reporting progress/last checkpoint related stats is a good
> idea IMO.
I agree with Andres: progress reporting isn't really quite the same
thing as stats, and either place seems like it could be reasonable. I
don't presently have an opinion on which is a better fit, but I don't
think the fact that CheckpointerShmemStruct is used for running
checkpoints rules anything out. Progress reporting is *also* about
running checkpoints. Any historical data you want to expose might not
be about running checkpoints, but, uh, so what? I don't really see
that as a strong argument against it fitting into this struct.
> I also think that a new pg_stat_checkpoint view is needed
> because, right now, the PgStat_CheckpointerStats stats are exposed via
> the pg_stat_bgwriter view, having a separate view for checkpoint stats
> is good here.
Yep.
> Also, removing CheckpointStatsData and moving all of
> those members to PgStat_CheckpointerStats, of course, by being careful
> about the amount of shared memory required, is also a good idea IMO.
> Going forward, PgStat_CheckpointerStats and pg_stat_checkpoint view
> can be a single point of location for all the checkpoint related
> stats.
I'm not sure that I completely follow this part, or that I agree with
it. I have never really understood why we drive background writer or
checkpointer statistics through the statistics collector. Here again,
for things like table statistics, there is no choice, because we could
have an unbounded number of tables and need to keep statistics about
all of them. The statistics collector can handle that by allocating
more memory as required. But there is only one background writer and
only one checkpointer, so that is not needed in those cases. Why not
just have them expose anything they want to expose through shared
memory directly?
If the statistics collector provides services that we care about, like
persisting data across restarts or making snapshots for transactional
behavior, then those might be reasons to go through it even for the
background writer or checkpointer. But if so, we should be explicit
about what the reasons are, both in the mailing list discussion and in
code comments. Otherwise I fear that we'll just end up doing something
in a more complicated way than is really necessary.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
@ 2022-11-16 19:52 ` Andres Freund <[email protected]>
2022-11-17 14:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
1 sibling, 1 reply; 85+ messages in thread
From: Andres Freund @ 2022-11-16 19:52 UTC (permalink / raw)
To: Robert Haas <[email protected]>; Melanie Plageman <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 2022-11-16 14:19:32 -0500, Robert Haas wrote:
> I have never really understood why we drive background writer or
> checkpointer statistics through the statistics collector.
To some degree it is required for durability - the stats system needs to know
how to write out those stats. But that wasn't ever a good reason to send
messages to the stats collector - it could just read the stats from shared
memory after all.
There's also integration with snapshots of the stats, resetting them, etc.
There's also the complexity that some of the stats e.g. for checkpointer
aren't about work the checkpointer did, but just have ended up there for
historical raisins. E.g. the number of fsyncs and writes done by backends.
See below:
> Here again, for things like table statistics, there is no choice, because we
> could have an unbounded number of tables and need to keep statistics about
> all of them. The statistics collector can handle that by allocating more
> memory as required. But there is only one background writer and only one
> checkpointer, so that is not needed in those cases. Why not just have them
> expose anything they want to expose through shared memory directly?
That's how it is in 15+. The memory for "fixed-numbered" or "global"
statistics are maintained by the stats system, but in plain shared memory,
allocated at server start. Not via the hash table.
Right now stats updates for the checkpointer use the "changecount" approach to
updates. For now that makes sense, because we update the stats only
occasionally (after a checkpoint or when writing in CheckpointWriteDelay()) -
a stats viewer seeing the checkpoint count go up, without yet seeing the
corresponding buffers written would be misleading.
I don't think we'd want every buffer write or whatnot go through the
changecount mechanism, on some non-x86 platforms that could be noticable. But
if we didn't stage the stats updates locally I think we could make most of the
stats changes without that overhead. For updates that just increment a single
counter there's simply no benefit in the changecount mechanism afaict.
I didn't want to do that change during the initial shared memory stats work,
it already was bigger than I could handle...
It's not quite clear to me what the best path forward is for
buf_written_backend / buf_fsync_backend, which currently are reported via the
checkpointer stats. I think the best path might be to stop counting them via
the CheckpointerShmem->num_backend_writes etc and just populate the fields in
the view (for backward compat) via the proposed [1] pg_stat_io patch. Doing
that accounting with CheckpointerCommLock held exclusively isn't free.
> If the statistics collector provides services that we care about, like
> persisting data across restarts or making snapshots for transactional
> behavior, then those might be reasons to go through it even for the
> background writer or checkpointer. But if so, we should be explicit
> about what the reasons are, both in the mailing list discussion and in
> code comments. Otherwise I fear that we'll just end up doing something
> in a more complicated way than is really necessary.
I tried to provide at least some of that in the comments at the start of
pgstat.c in 15+. There's very likely more that should be added, but I think
it's a decent start.
Greetings,
Andres Freund
[1] https://www.postgresql.org/message-id/CAOtHd0ApHna7_p6mvHoO%2BgLZdxjaQPRemg3_o0a4ytCPijLytQ%40mail.g...
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 19:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
@ 2022-11-17 14:03 ` Robert Haas <[email protected]>
2022-11-17 16:24 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Robert Haas @ 2022-11-17 14:03 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Bharath Rupireddy <[email protected]>; Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Nov 16, 2022 at 2:52 PM Andres Freund <[email protected]> wrote:
> I don't think we'd want every buffer write or whatnot go through the
> changecount mechanism, on some non-x86 platforms that could be noticable. But
> if we didn't stage the stats updates locally I think we could make most of the
> stats changes without that overhead. For updates that just increment a single
> counter there's simply no benefit in the changecount mechanism afaict.
You might be right, but I'm not sure whether it's worth stressing
about. The progress reporting mechanism uses the st_changecount
mechanism, too, and as far as I know nobody's complained about that
having too much overhead. Maybe they have, though, and I've just
missed it.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 19:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-17 14:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
@ 2022-11-17 16:24 ` Andres Freund <[email protected]>
2022-11-17 17:18 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Tom Lane <[email protected]>
2022-11-17 17:21 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
0 siblings, 2 replies; 85+ messages in thread
From: Andres Freund @ 2022-11-17 16:24 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Bharath Rupireddy <[email protected]>; Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 2022-11-17 09:03:32 -0500, Robert Haas wrote:
> On Wed, Nov 16, 2022 at 2:52 PM Andres Freund <[email protected]> wrote:
> > I don't think we'd want every buffer write or whatnot go through the
> > changecount mechanism, on some non-x86 platforms that could be noticable. But
> > if we didn't stage the stats updates locally I think we could make most of the
> > stats changes without that overhead. For updates that just increment a single
> > counter there's simply no benefit in the changecount mechanism afaict.
>
> You might be right, but I'm not sure whether it's worth stressing
> about. The progress reporting mechanism uses the st_changecount
> mechanism, too, and as far as I know nobody's complained about that
> having too much overhead. Maybe they have, though, and I've just
> missed it.
I've seen it in profiles, although not as the major contributor. Most places
do a reasonable amount of work between calls though.
As an experiment, I added a progress report to BufferSync()'s first loop
(i.e. where it checks all buffers). On a 128GB shared_buffers cluster that
increases the time for a do-nothing checkpoint from ~235ms to ~280ms. If I
remove the changecount stuff and use a single write + write barrier, it ends
up as 250ms. Inlining brings it down a bit further, to 247ms.
Obviously this is a very extreme case - we only do very little work between
the progress report calls. But it does seem to show that the overhead is not
entirely neglegible.
I think pgstat_progress_start_command() needs the changecount stuff, as does
pgstat_progress_update_multi_param(). But for anything updating a single
parameter at a time it really doesn't do anything useful on a platform that
doesn't tear 64bit writes (so it could be #ifdef
PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY).
Out of further curiosity I wanted to test the impact when the loop doesn't
even do a LockBufHdr() and added an unlocked pre-check. 109ms without
progress. 138ms with. 114ms with the simplified
pgstat_progress_update_param(). 108ms after inlining the simplified
pgstat_progress_update_param().
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 19:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-17 14:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-17 16:24 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
@ 2022-11-17 17:18 ` Tom Lane <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Tom Lane @ 2022-11-17 17:18 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Robert Haas <[email protected]>; Melanie Plageman <[email protected]>; Bharath Rupireddy <[email protected]>; Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Andres Freund <[email protected]> writes:
> I think pgstat_progress_start_command() needs the changecount stuff, as does
> pgstat_progress_update_multi_param(). But for anything updating a single
> parameter at a time it really doesn't do anything useful on a platform that
> doesn't tear 64bit writes (so it could be #ifdef
> PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY).
Seems safe to restrict it to that case.
regards, tom lane
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 19:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-17 14:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-17 16:24 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
@ 2022-11-17 17:21 ` Robert Haas <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Robert Haas @ 2022-11-17 17:21 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Melanie Plageman <[email protected]>; Bharath Rupireddy <[email protected]>; Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, Nov 17, 2022 at 11:24 AM Andres Freund <[email protected]> wrote:
> As an experiment, I added a progress report to BufferSync()'s first loop
> (i.e. where it checks all buffers). On a 128GB shared_buffers cluster that
> increases the time for a do-nothing checkpoint from ~235ms to ~280ms. If I
> remove the changecount stuff and use a single write + write barrier, it ends
> up as 250ms. Inlining brings it down a bit further, to 247ms.
OK, I'd say that's pretty good evidence that we can't totally
disregard the issue.
--
Robert Haas
EDB: http://www.enterprisedb.com
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
@ 2022-11-17 13:31 ` Bharath Rupireddy <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Bharath Rupireddy @ 2022-11-17 13:31 UTC (permalink / raw)
To: Robert Haas <[email protected]>; +Cc: Drouvot, Bertrand <[email protected]>; Nitin Jadhav <[email protected]>; Andres Freund <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, Nov 17, 2022 at 12:49 AM Robert Haas <[email protected]> wrote:
>
> > I also think that a new pg_stat_checkpoint view is needed
> > because, right now, the PgStat_CheckpointerStats stats are exposed via
> > the pg_stat_bgwriter view, having a separate view for checkpoint stats
> > is good here.
>
> Yep.
On Wed, Nov 16, 2022 at 11:44 PM Andres Freund <[email protected]> wrote:
>
> > I also think that a new pg_stat_checkpoint view is needed
> > because, right now, the PgStat_CheckpointerStats stats are exposed via
> > the pg_stat_bgwriter view, having a separate view for checkpoint stats
> > is good here.
>
> I agree that we should do that, but largely independent of the architectural
> question at hand.
Thanks. I quickly prepared a patch introducing pg_stat_checkpointer
view and posted it here -
https://www.postgresql.org/message-id/CALj2ACVxX2ii%3D66RypXRweZe2EsBRiPMj0aHfRfHUeXJcC7kHg%40mail.g....
--
Bharath Rupireddy
PostgreSQL Contributors Team
RDS Open Source Databases
Amazon Web Services: https://aws.amazon.com
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
@ 2022-11-15 20:18 ` Andres Freund <[email protected]>
3 siblings, 0 replies; 85+ messages in thread
From: Andres Freund @ 2022-11-15 20:18 UTC (permalink / raw)
To: Drouvot, Bertrand <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 2022-11-04 09:25:52 +0100, Drouvot, Bertrand wrote:
>
> @@ -7023,29 +7048,63 @@ static void
> CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
> {
> CheckPointRelationMap();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
> CheckPointReplicationSlots();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
> CheckPointSnapBuild();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
> CheckPointLogicalRewriteHeap();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN);
> CheckPointReplicationOrigin();
>
> /* Write out all dirty data in SLRUs and the main buffer pool */
> TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
> CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
> CheckPointCLOG();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
> CheckPointCommitTs();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
> CheckPointSUBTRANS();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
> CheckPointMultiXact();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES);
> CheckPointPredicate();
> +
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_BUFFERS);
> CheckPointBuffers(flags);
>
> /* Perform all queued up fsyncs */
> TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
> CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_SYNC_FILES);
> ProcessSyncRequests();
> CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
> TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
>
> /* We deliberately delay 2PC checkpointing as long as possible */
> + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> + PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
> CheckPointTwoPhase(checkPointRedo);
> }
This is quite the code bloat. Can we make this less duplicative?
> +CREATE VIEW pg_stat_progress_checkpoint AS
> + SELECT
> + S.pid AS pid,
> + CASE S.param1 WHEN 1 THEN 'checkpoint'
> + WHEN 2 THEN 'restartpoint'
> + END AS type,
> + ( CASE WHEN (S.param2 & 4) > 0 THEN 'immediate ' ELSE '' END ||
> + CASE WHEN (S.param2 & 8) > 0 THEN 'force ' ELSE '' END ||
> + CASE WHEN (S.param2 & 16) > 0 THEN 'flush-all ' ELSE '' END ||
> + CASE WHEN (S.param2 & 32) > 0 THEN 'wait ' ELSE '' END ||
> + CASE WHEN (S.param2 & 128) > 0 THEN 'wal ' ELSE '' END ||
> + CASE WHEN (S.param2 & 256) > 0 THEN 'time ' ELSE '' END
> + ) AS flags,
> + ( '0/0'::pg_lsn +
> + ((CASE
> + WHEN S.param3 < 0 THEN pow(2::numeric, 64::numeric)::numeric
> + ELSE 0::numeric
> + END) +
> + S.param3::numeric)
> + ) AS start_lsn,
I don't think we should embed this much complexity in the view
defintions. It's hard to read, bloats the catalog, we can't fix them once
released. This stuff seems like it should be in a helper function.
I don't have any iea what that pow stuff is supposed to be doing.
> + to_timestamp(946684800 + (S.param4::float8 / 1000000)) AS start_time,
I don't think this is a reasonable path - embedding way too much low-level
details about the timestamp format in the view definition. Why do we need to
do this?
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
@ 2022-12-07 19:03 ` Andres Freund <[email protected]>
3 siblings, 0 replies; 85+ messages in thread
From: Andres Freund @ 2022-12-07 19:03 UTC (permalink / raw)
To: Drouvot, Bertrand <[email protected]>; +Cc: Nitin Jadhav <[email protected]>; Bharath Rupireddy <[email protected]>; Matthias van de Meent <[email protected]>; Ashutosh Sharma <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 2022-11-04 09:25:52 +0100, Drouvot, Bertrand wrote:
> Please find attached a rebase in v7.
cfbot complains that the docs don't build:
https://cirrus-ci.com/task/6694349031866368?logs=docs_build#L296
[03:24:27.317] ref/checkpoint.sgml:66: element para: validity error : Element para is not declared in para list of possible children
I've marked the patch as waitin-on-author for now.
There's been a bunch of architectural feedback too, but tbh, I don't know if
we came to any conclusion on that front...
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-03 15:00 ` Ashutosh Sharma <[email protected]>
2022-03-07 14:45 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2 siblings, 1 reply; 85+ messages in thread
From: Ashutosh Sharma @ 2022-03-03 15:00 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Here are some of my review comments on the latest patch:
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>kind</structfield> <type>text</type>
+ </para>
+ <para>
+ Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
+ </para></entry>
+ </row>
This looks a bit confusing. Two columns, one with the name "checkpoint
types" and another "checkpoint kinds". You can probably rename
checkpoint-kinds to checkpoint-flags and let the checkpoint-types be
as-it-is.
==
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the checkpoint.
Let's make this message consistent with the already existing message
for pg_stat_wal_receiver. See description for pg_stat_wal_receiver
view in "Dynamic Statistics Views" table.
==
[local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
-[ RECORD 1 ]-----+-------------------------------------
pid | 22043
type | checkpoint
kind | immediate force wait requested time
I think the output in the kind column can be displayed as {immediate,
force, wait, requested, time}. By the way these are all checkpoint
flags so it is better to display it as checkpoint flags instead of
checkpoint kind as mentioned in one of my previous comments.
==
[local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
-[ RECORD 1 ]-----+-------------------------------------
pid | 22043
type | checkpoint
kind | immediate force wait requested time
start_lsn | 0/14C60F8
start_time | 2022-03-03 18:59:56.018662+05:30
phase | performing two phase checkpoint
This is the output I see when the checkpointer process has come out of
the two phase checkpoint and is currently writing checkpoint xlog
records and doing other stuff like updating control files etc. Is this
okay?
==
The output of log_checkpoint shows the number of buffers written is 3
whereas the output of pg_stat_progress_checkpoint shows it as 0. See
below:
2022-03-03 20:04:45.643 IST [22043] LOG: checkpoint complete: wrote 3
buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled;
write=24.652 s, sync=104.256 s, total=3889.625 s; sync files=2,
longest=0.011 s, average=0.008 s; distance=0 kB, estimate=0 kB
--
[local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
-[ RECORD 1 ]-----+-------------------------------------
pid | 22043
type | checkpoint
kind | immediate force wait requested time
start_lsn | 0/14C60F8
start_time | 2022-03-03 18:59:56.018662+05:30
phase | finalizing
buffers_total | 0
buffers_processed | 0
buffers_written | 0
Any idea why this mismatch?
==
I think we can add a couple of more information to this view -
start_time for buffer write operation and start_time for buffer sync
operation. These are two very time consuming tasks in a checkpoint and
people would find it useful to know how much time is being taken by
the checkpoint in I/O operation phase. thoughts?
--
With Regards,
Ashutosh Sharma.
On Wed, Mar 2, 2022 at 4:45 PM Nitin Jadhav
<[email protected]> wrote:
>
> Thanks for reviewing.
>
> > > > I suggested upthread to store the starting timeline instead. This way you can
> > > > deduce whether it's a restartpoint or a checkpoint, but you can also deduce
> > > > other information, like what was the starting WAL.
> > >
> > > I don't understand why we need the timeline here to just determine
> > > whether it's a restartpoint or checkpoint.
> >
> > I'm not saying it's necessary, I'm saying that for the same space usage we can
> > store something a bit more useful. If no one cares about having the starting
> > timeline available for no extra cost then sure, let's just store the kind
> > directly.
>
> Fixed.
>
> > 2) Can't we just have these checks inside CASE-WHEN-THEN-ELSE blocks
> > directly instead of new function pg_stat_get_progress_checkpoint_kind?
> > + snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
> > + (flags == 0) ? "unknown" : "",
> > + (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
> > + (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
> > + (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
> > + (flags & CHECKPOINT_FORCE) ? "force " : "",
> > + (flags & CHECKPOINT_WAIT) ? "wait " : "",
> > + (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
> > + (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
> > + (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
>
> Fixed.
> ---
>
> > 5) Do we need a special phase for this checkpoint operation? I'm not
> > sure in which cases it will take a long time, but it looks like
> > there's a wait loop here.
> > vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
> > if (nvxids > 0)
> > {
> > do
> > {
> > pg_usleep(10000L); /* wait for 10 msec */
> > } while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids));
> > }
>
> Yes. It is better to add a separate phase here.
> ---
>
> > Also, how about special phases for SyncPostCheckpoint(),
> > SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
> > PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
> > it might be increase in future (?)), TruncateSUBTRANS()?
>
> SyncPreCheckpoint() is just incrementing a counter and
> PreallocXlogFiles() currently pre-allocates only 1 WAL file. I feel
> there is no need to add any phases for these as of now. We can add in
> the future if necessary. Added phases for SyncPostCheckpoint(),
> InvalidateObsoleteReplicationSlots() and TruncateSUBTRANS().
> ---
>
> > 6) SLRU (Simple LRU) isn't a phase here, you can just say
> > PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES.
> > +
> > + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> > + PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
> > CheckPointPredicate();
> >
> > And :s/checkpointing SLRU pages/checkpointing predicate lock pages
> >+ WHEN 9 THEN 'checkpointing SLRU pages'
>
> Fixed.
> ---
>
> > 7) :s/PROGRESS_CHECKPOINT_PHASE_FILE_SYNC/PROGRESS_CHECKPOINT_PHASE_PROCESS_FILE_SYNC_REQUESTS
>
> I feel PROGRESS_CHECKPOINT_PHASE_FILE_SYNC is a better option here as
> it describes the purpose in less words.
>
> > And :s/WHEN 11 THEN 'performing sync requests'/WHEN 11 THEN
> > 'processing file sync requests'
>
> Fixed.
> ---
>
> > 8) :s/Finalizing/finalizing
> > + WHEN 14 THEN 'Finalizing'
>
> Fixed.
> ---
>
> > 9) :s/checkpointing snapshots/checkpointing logical replication snapshot files
> > + WHEN 3 THEN 'checkpointing snapshots'
> > :s/checkpointing logical rewrite mappings/checkpointing logical
> > replication rewrite mapping files
> > + WHEN 4 THEN 'checkpointing logical rewrite mappings'
>
> Fixed.
> ---
>
> > 10) I'm not sure if it's discussed, how about adding the number of
> > snapshot/mapping files so far the checkpoint has processed in file
> > processing while loops of
> > CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
> > be many logical snapshot or mapping files and users may be interested
> > in knowing the so-far-processed-file-count.
>
> I had thought about this while sharing the v1 patch and mentioned my
> views upthread. I feel it won't give meaningful progress information
> (It can be treated as statistics). Hence not included. Thoughts?
>
> > > > As mentioned upthread, there can be multiple backends that request a
> > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > of backend that are waiting for a new checkpoint.
> > >
> > > Yeah, you are right. Let's not go that path and store an array of
> > > pids. I don't see a strong use-case with the pid of the process
> > > requesting checkpoint. If required, we can add it later once the
> > > pg_stat_progress_checkpoint view gets in.
> >
> > I don't think that's really necessary to give the pid list.
> >
> > If you requested a new checkpoint, it doesn't matter if it's only your backend
> > that triggered it, another backend or a few other dozen, the result will be the
> > same and you have the information that the request has been seen. We could
> > store just a bool for that but having a number instead also gives a bit more
> > information and may allow you to detect some broken logic on your client code
> > if it keeps increasing.
>
> It's a good metric to show in the view but the information is not
> readily available. Additional code is required to calculate the number
> of requests. Is it worth doing that? I feel this can be added later if
> required.
>
> Please find the v4 patch attached and share your thoughts.
>
> Thanks & Regards,
> Nitin Jadhav
>
> On Tue, Mar 1, 2022 at 2:27 PM Nitin Jadhav
> <[email protected]> wrote:
> >
> > > > 3) Why do we need this extra calculation for start_lsn?
> > > > Do you ever see a negative LSN or something here?
> > > > + ('0/0'::pg_lsn + (
> > > > + CASE
> > > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > > + ELSE (0)::numeric
> > > > + END + (s.param3)::numeric)) AS start_lsn,
> > >
> > > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > > bigint type; thus the signed int64. This cast is OK as it wraps
> > > around, but that means we have to take care to correctly display the
> > > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > > the special-casing for negative values.
> >
> > Yes. The extra calculation is required here as we are storing unit64
> > value in the variable of type int64. When we convert uint64 to int64
> > then the bit pattern is preserved (so no data is lost). The high-order
> > bit becomes the sign bit and if the sign bit is set, both the sign and
> > magnitude of the value changes. To safely get the actual uint64 value
> > whatever was assigned, we need the above calculations.
> >
> > > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > > the reasoning for having this function and it's named as *checkpoint*
> > > > when it doesn't do anything specific to the checkpoint at all?
> > >
> > > I hadn't thought of using the types' inout functions, but it looks
> > > like timestamp IO functions use a formatted timestring, which won't
> > > work with the epoch-based timestamp stored in the view.
> >
> > There is a variation of to_timestamp() which takes UNIX epoch (float8)
> > as an argument and converts it to timestamptz but we cannot directly
> > call this function with S.param4.
> >
> > TimestampTz
> > GetCurrentTimestamp(void)
> > {
> > TimestampTz result;
> > struct timeval tp;
> >
> > gettimeofday(&tp, NULL);
> >
> > result = (TimestampTz) tp.tv_sec -
> > ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY);
> > result = (result * USECS_PER_SEC) + tp.tv_usec;
> >
> > return result;
> > }
> >
> > S.param4 contains the output of the above function
> > (GetCurrentTimestamp()) which returns Postgres epoch but the
> > to_timestamp() expects UNIX epoch as input. So some calculation is
> > required here. I feel the SQL 'to_timestamp(946684800 +
> > (S.param4::float / 1000000)) AS start_time' works fine. The value
> > '946684800' is equal to ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) *
> > SECS_PER_DAY). I am not sure whether it is good practice to use this
> > way. Kindly share your thoughts.
> >
> > Thanks & Regards,
> > Nitin Jadhav
> >
> > On Mon, Feb 28, 2022 at 6:40 PM Matthias van de Meent
> > <[email protected]> wrote:
> > >
> > > On Sun, 27 Feb 2022 at 16:14, Bharath Rupireddy
> > > <[email protected]> wrote:
> > > > 3) Why do we need this extra calculation for start_lsn?
> > > > Do you ever see a negative LSN or something here?
> > > > + ('0/0'::pg_lsn + (
> > > > + CASE
> > > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > > + ELSE (0)::numeric
> > > > + END + (s.param3)::numeric)) AS start_lsn,
> > >
> > > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > > bigint type; thus the signed int64. This cast is OK as it wraps
> > > around, but that means we have to take care to correctly display the
> > > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > > the special-casing for negative values.
> > >
> > > As to whether it is reasonable: Generating 16GB of wal every second
> > > (2^34 bytes /sec) is probably not impossible (cpu <> memory bandwidth
> > > has been > 20GB/sec for a while); and that leaves you 2^29 seconds of
> > > database runtime; or about 17 years. Seeing that a cluster can be
> > > `pg_upgrade`d (which doesn't reset cluster LSN) since PG 9.0 from at
> > > least version PG 8.4.0 (2009) (and through pg_migrator, from 8.3.0)),
> > > we can assume that clusters hitting LSN=2^63 will be a reasonable
> > > possibility within the next few years. As the lifespan of a PG release
> > > is about 5 years, it doesn't seem impossible that there will be actual
> > > clusters that are going to hit this naturally in the lifespan of PG15.
> > >
> > > It is also possible that someone fat-fingers pg_resetwal; and creates
> > > a cluster with LSN >= 2^63; resulting in negative values in the
> > > s.param3 field. Not likely, but we can force such situations; and as
> > > such we should handle that gracefully.
> > >
> > > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > > the reasoning for having this function and it's named as *checkpoint*
> > > > when it doesn't do anything specific to the checkpoint at all?
> > >
> > > I hadn't thought of using the types' inout functions, but it looks
> > > like timestamp IO functions use a formatted timestring, which won't
> > > work with the epoch-based timestamp stored in the view.
> > >
> > > > Having 3 unnecessary functions that aren't useful to the users at all
> > > > in proc.dat will simply eatup the function oids IMO. Hence, I suggest
> > > > let's try to do without extra functions.
> > >
> > > I agree that (1) could be simplified, or at least fully expressed in
> > > SQL without exposing too many internals. If we're fine with exposing
> > > internals like flags and type layouts, then (2), and arguably (4), can
> > > be expressed in SQL as well.
> > >
> > > -Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-03 15:00 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
@ 2022-03-07 14:45 ` Nitin Jadhav <[email protected]>
2022-03-08 15:00 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-03-07 14:45 UTC (permalink / raw)
To: Ashutosh Sharma <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> + <row>
> + <entry role="catalog_table_entry"><para role="column_definition">
> + <structfield>type</structfield> <type>text</type>
> + </para>
> + <para>
> + Type of checkpoint. See <xref linkend="checkpoint-types"/>.
> + </para></entry>
> + </row>
> +
> + <row>
> + <entry role="catalog_table_entry"><para role="column_definition">
> + <structfield>kind</structfield> <type>text</type>
> + </para>
> + <para>
> + Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
> + </para></entry>
> + </row>
>
> This looks a bit confusing. Two columns, one with the name "checkpoint
> types" and another "checkpoint kinds". You can probably rename
> checkpoint-kinds to checkpoint-flags and let the checkpoint-types be
> as-it-is.
Makes sense. I will change in the next patch.
---
> + <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
> + <entry>One row only, showing the progress of the checkpoint.
>
> Let's make this message consistent with the already existing message
> for pg_stat_wal_receiver. See description for pg_stat_wal_receiver
> view in "Dynamic Statistics Views" table.
You want me to change "One row only" to "Only one row" ? If that is
the case then for other views in the "Collected Statistics Views"
table, it is referred as "One row only". Let me know if you are
pointing out something else.
---
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
>
> I think the output in the kind column can be displayed as {immediate,
> force, wait, requested, time}. By the way these are all checkpoint
> flags so it is better to display it as checkpoint flags instead of
> checkpoint kind as mentioned in one of my previous comments.
I will update in the next patch.
---
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
> start_lsn | 0/14C60F8
> start_time | 2022-03-03 18:59:56.018662+05:30
> phase | performing two phase checkpoint
>
> This is the output I see when the checkpointer process has come out of
> the two phase checkpoint and is currently writing checkpoint xlog
> records and doing other stuff like updating control files etc. Is this
> okay?
The idea behind choosing the phases is based on the functionality
which takes longer time to execute. Since after two phase checkpoint
till post checkpoint cleanup won't take much time to execute, I have
not added any additional phase for that. But I also agree that this
gives wrong information to the user. How about mentioning the phase
information at the end of each phase like "Initializing",
"Initialization done", ..., "two phase checkpoint done", "post
checkpoint cleanup done", .., "finalizing". Except for the first phase
("initializing") and last phase ("finalizing"), all the other phases
describe the end of a certain operation. I feel this gives correct
information even though the phase name/description does not represent
the entire code block between two phases. For example if the current
phase is ''two phase checkpoint done". Then the user can infer that
the checkpointer has done till two phase checkpoint and it is doing
other stuff that are after that. Thoughts?
---
> The output of log_checkpoint shows the number of buffers written is 3
> whereas the output of pg_stat_progress_checkpoint shows it as 0. See
> below:
>
> 2022-03-03 20:04:45.643 IST [22043] LOG: checkpoint complete: wrote 3
> buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled;
> write=24.652 s, sync=104.256 s, total=3889.625 s; sync files=2,
> longest=0.011 s, average=0.008 s; distance=0 kB, estimate=0 kB
>
> --
>
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
> start_lsn | 0/14C60F8
> start_time | 2022-03-03 18:59:56.018662+05:30
> phase | finalizing
> buffers_total | 0
> buffers_processed | 0
> buffers_written | 0
>
> Any idea why this mismatch?
Good catch. In BufferSync() we have 'num_to_scan' (buffers_total)
which indicates the total number of buffers to be processed. Based on
that, the 'buffers_processed' and 'buffers_written' counter gets
incremented. I meant these values may reach upto 'buffers_total'. The
current pg_stat_progress_view support above information. There is
another place when 'ckpt_bufs_written' gets incremented (In
SlruInternalWritePage()). This increment is above the 'buffers_total'
value and it is included in the server log message (checkpoint end)
and not included in the view. I am a bit confused here. If we include
this increment in the view then we cannot calculate the exact
'buffers_total' beforehand. Can we increment the 'buffers_toal' also
when 'ckpt_bufs_written' gets incremented so that we can match the
behaviour with checkpoint end message? Please share your thoughts.
---
> I think we can add a couple of more information to this view -
> start_time for buffer write operation and start_time for buffer sync
> operation. These are two very time consuming tasks in a checkpoint and
> people would find it useful to know how much time is being taken by
> the checkpoint in I/O operation phase. thoughts?
The detailed progress is getting shown for these 2 phases of the
checkpoint like 'buffers_processed', 'buffers_written' and
'files_synced'. Hence I did not think about adding start time and If
it is really required, then I can add.
Thanks & Regards,
Nitin Jadhav
On Thu, Mar 3, 2022 at 8:30 PM Ashutosh Sharma <[email protected]> wrote:
>
> Here are some of my review comments on the latest patch:
>
> + <row>
> + <entry role="catalog_table_entry"><para role="column_definition">
> + <structfield>type</structfield> <type>text</type>
> + </para>
> + <para>
> + Type of checkpoint. See <xref linkend="checkpoint-types"/>.
> + </para></entry>
> + </row>
> +
> + <row>
> + <entry role="catalog_table_entry"><para role="column_definition">
> + <structfield>kind</structfield> <type>text</type>
> + </para>
> + <para>
> + Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
> + </para></entry>
> + </row>
>
> This looks a bit confusing. Two columns, one with the name "checkpoint
> types" and another "checkpoint kinds". You can probably rename
> checkpoint-kinds to checkpoint-flags and let the checkpoint-types be
> as-it-is.
>
> ==
>
> + <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
> + <entry>One row only, showing the progress of the checkpoint.
>
> Let's make this message consistent with the already existing message
> for pg_stat_wal_receiver. See description for pg_stat_wal_receiver
> view in "Dynamic Statistics Views" table.
>
> ==
>
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
>
> I think the output in the kind column can be displayed as {immediate,
> force, wait, requested, time}. By the way these are all checkpoint
> flags so it is better to display it as checkpoint flags instead of
> checkpoint kind as mentioned in one of my previous comments.
>
> ==
>
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
> start_lsn | 0/14C60F8
> start_time | 2022-03-03 18:59:56.018662+05:30
> phase | performing two phase checkpoint
>
>
> This is the output I see when the checkpointer process has come out of
> the two phase checkpoint and is currently writing checkpoint xlog
> records and doing other stuff like updating control files etc. Is this
> okay?
>
> ==
>
> The output of log_checkpoint shows the number of buffers written is 3
> whereas the output of pg_stat_progress_checkpoint shows it as 0. See
> below:
>
> 2022-03-03 20:04:45.643 IST [22043] LOG: checkpoint complete: wrote 3
> buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled;
> write=24.652 s, sync=104.256 s, total=3889.625 s; sync files=2,
> longest=0.011 s, average=0.008 s; distance=0 kB, estimate=0 kB
>
> --
>
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
> start_lsn | 0/14C60F8
> start_time | 2022-03-03 18:59:56.018662+05:30
> phase | finalizing
> buffers_total | 0
> buffers_processed | 0
> buffers_written | 0
>
> Any idea why this mismatch?
>
> ==
>
> I think we can add a couple of more information to this view -
> start_time for buffer write operation and start_time for buffer sync
> operation. These are two very time consuming tasks in a checkpoint and
> people would find it useful to know how much time is being taken by
> the checkpoint in I/O operation phase. thoughts?
>
> --
> With Regards,
> Ashutosh Sharma.
>
> On Wed, Mar 2, 2022 at 4:45 PM Nitin Jadhav
> <[email protected]> wrote:
> >
> > Thanks for reviewing.
> >
> > > > > I suggested upthread to store the starting timeline instead. This way you can
> > > > > deduce whether it's a restartpoint or a checkpoint, but you can also deduce
> > > > > other information, like what was the starting WAL.
> > > >
> > > > I don't understand why we need the timeline here to just determine
> > > > whether it's a restartpoint or checkpoint.
> > >
> > > I'm not saying it's necessary, I'm saying that for the same space usage we can
> > > store something a bit more useful. If no one cares about having the starting
> > > timeline available for no extra cost then sure, let's just store the kind
> > > directly.
> >
> > Fixed.
> >
> > > 2) Can't we just have these checks inside CASE-WHEN-THEN-ELSE blocks
> > > directly instead of new function pg_stat_get_progress_checkpoint_kind?
> > > + snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
> > > + (flags == 0) ? "unknown" : "",
> > > + (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
> > > + (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
> > > + (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
> > > + (flags & CHECKPOINT_FORCE) ? "force " : "",
> > > + (flags & CHECKPOINT_WAIT) ? "wait " : "",
> > > + (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
> > > + (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
> > > + (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
> >
> > Fixed.
> > ---
> >
> > > 5) Do we need a special phase for this checkpoint operation? I'm not
> > > sure in which cases it will take a long time, but it looks like
> > > there's a wait loop here.
> > > vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
> > > if (nvxids > 0)
> > > {
> > > do
> > > {
> > > pg_usleep(10000L); /* wait for 10 msec */
> > > } while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids));
> > > }
> >
> > Yes. It is better to add a separate phase here.
> > ---
> >
> > > Also, how about special phases for SyncPostCheckpoint(),
> > > SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
> > > PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
> > > it might be increase in future (?)), TruncateSUBTRANS()?
> >
> > SyncPreCheckpoint() is just incrementing a counter and
> > PreallocXlogFiles() currently pre-allocates only 1 WAL file. I feel
> > there is no need to add any phases for these as of now. We can add in
> > the future if necessary. Added phases for SyncPostCheckpoint(),
> > InvalidateObsoleteReplicationSlots() and TruncateSUBTRANS().
> > ---
> >
> > > 6) SLRU (Simple LRU) isn't a phase here, you can just say
> > > PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES.
> > > +
> > > + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> > > + PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
> > > CheckPointPredicate();
> > >
> > > And :s/checkpointing SLRU pages/checkpointing predicate lock pages
> > >+ WHEN 9 THEN 'checkpointing SLRU pages'
> >
> > Fixed.
> > ---
> >
> > > 7) :s/PROGRESS_CHECKPOINT_PHASE_FILE_SYNC/PROGRESS_CHECKPOINT_PHASE_PROCESS_FILE_SYNC_REQUESTS
> >
> > I feel PROGRESS_CHECKPOINT_PHASE_FILE_SYNC is a better option here as
> > it describes the purpose in less words.
> >
> > > And :s/WHEN 11 THEN 'performing sync requests'/WHEN 11 THEN
> > > 'processing file sync requests'
> >
> > Fixed.
> > ---
> >
> > > 8) :s/Finalizing/finalizing
> > > + WHEN 14 THEN 'Finalizing'
> >
> > Fixed.
> > ---
> >
> > > 9) :s/checkpointing snapshots/checkpointing logical replication snapshot files
> > > + WHEN 3 THEN 'checkpointing snapshots'
> > > :s/checkpointing logical rewrite mappings/checkpointing logical
> > > replication rewrite mapping files
> > > + WHEN 4 THEN 'checkpointing logical rewrite mappings'
> >
> > Fixed.
> > ---
> >
> > > 10) I'm not sure if it's discussed, how about adding the number of
> > > snapshot/mapping files so far the checkpoint has processed in file
> > > processing while loops of
> > > CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
> > > be many logical snapshot or mapping files and users may be interested
> > > in knowing the so-far-processed-file-count.
> >
> > I had thought about this while sharing the v1 patch and mentioned my
> > views upthread. I feel it won't give meaningful progress information
> > (It can be treated as statistics). Hence not included. Thoughts?
> >
> > > > > As mentioned upthread, there can be multiple backends that request a
> > > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > > of backend that are waiting for a new checkpoint.
> > > >
> > > > Yeah, you are right. Let's not go that path and store an array of
> > > > pids. I don't see a strong use-case with the pid of the process
> > > > requesting checkpoint. If required, we can add it later once the
> > > > pg_stat_progress_checkpoint view gets in.
> > >
> > > I don't think that's really necessary to give the pid list.
> > >
> > > If you requested a new checkpoint, it doesn't matter if it's only your backend
> > > that triggered it, another backend or a few other dozen, the result will be the
> > > same and you have the information that the request has been seen. We could
> > > store just a bool for that but having a number instead also gives a bit more
> > > information and may allow you to detect some broken logic on your client code
> > > if it keeps increasing.
> >
> > It's a good metric to show in the view but the information is not
> > readily available. Additional code is required to calculate the number
> > of requests. Is it worth doing that? I feel this can be added later if
> > required.
> >
> > Please find the v4 patch attached and share your thoughts.
> >
> > Thanks & Regards,
> > Nitin Jadhav
> >
> > On Tue, Mar 1, 2022 at 2:27 PM Nitin Jadhav
> > <[email protected]> wrote:
> > >
> > > > > 3) Why do we need this extra calculation for start_lsn?
> > > > > Do you ever see a negative LSN or something here?
> > > > > + ('0/0'::pg_lsn + (
> > > > > + CASE
> > > > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > > > + ELSE (0)::numeric
> > > > > + END + (s.param3)::numeric)) AS start_lsn,
> > > >
> > > > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > > > bigint type; thus the signed int64. This cast is OK as it wraps
> > > > around, but that means we have to take care to correctly display the
> > > > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > > > the special-casing for negative values.
> > >
> > > Yes. The extra calculation is required here as we are storing unit64
> > > value in the variable of type int64. When we convert uint64 to int64
> > > then the bit pattern is preserved (so no data is lost). The high-order
> > > bit becomes the sign bit and if the sign bit is set, both the sign and
> > > magnitude of the value changes. To safely get the actual uint64 value
> > > whatever was assigned, we need the above calculations.
> > >
> > > > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > > > the reasoning for having this function and it's named as *checkpoint*
> > > > > when it doesn't do anything specific to the checkpoint at all?
> > > >
> > > > I hadn't thought of using the types' inout functions, but it looks
> > > > like timestamp IO functions use a formatted timestring, which won't
> > > > work with the epoch-based timestamp stored in the view.
> > >
> > > There is a variation of to_timestamp() which takes UNIX epoch (float8)
> > > as an argument and converts it to timestamptz but we cannot directly
> > > call this function with S.param4.
> > >
> > > TimestampTz
> > > GetCurrentTimestamp(void)
> > > {
> > > TimestampTz result;
> > > struct timeval tp;
> > >
> > > gettimeofday(&tp, NULL);
> > >
> > > result = (TimestampTz) tp.tv_sec -
> > > ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY);
> > > result = (result * USECS_PER_SEC) + tp.tv_usec;
> > >
> > > return result;
> > > }
> > >
> > > S.param4 contains the output of the above function
> > > (GetCurrentTimestamp()) which returns Postgres epoch but the
> > > to_timestamp() expects UNIX epoch as input. So some calculation is
> > > required here. I feel the SQL 'to_timestamp(946684800 +
> > > (S.param4::float / 1000000)) AS start_time' works fine. The value
> > > '946684800' is equal to ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) *
> > > SECS_PER_DAY). I am not sure whether it is good practice to use this
> > > way. Kindly share your thoughts.
> > >
> > > Thanks & Regards,
> > > Nitin Jadhav
> > >
> > > On Mon, Feb 28, 2022 at 6:40 PM Matthias van de Meent
> > > <[email protected]> wrote:
> > > >
> > > > On Sun, 27 Feb 2022 at 16:14, Bharath Rupireddy
> > > > <[email protected]> wrote:
> > > > > 3) Why do we need this extra calculation for start_lsn?
> > > > > Do you ever see a negative LSN or something here?
> > > > > + ('0/0'::pg_lsn + (
> > > > > + CASE
> > > > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > > > + ELSE (0)::numeric
> > > > > + END + (s.param3)::numeric)) AS start_lsn,
> > > >
> > > > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > > > bigint type; thus the signed int64. This cast is OK as it wraps
> > > > around, but that means we have to take care to correctly display the
> > > > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > > > the special-casing for negative values.
> > > >
> > > > As to whether it is reasonable: Generating 16GB of wal every second
> > > > (2^34 bytes /sec) is probably not impossible (cpu <> memory bandwidth
> > > > has been > 20GB/sec for a while); and that leaves you 2^29 seconds of
> > > > database runtime; or about 17 years. Seeing that a cluster can be
> > > > `pg_upgrade`d (which doesn't reset cluster LSN) since PG 9.0 from at
> > > > least version PG 8.4.0 (2009) (and through pg_migrator, from 8.3.0)),
> > > > we can assume that clusters hitting LSN=2^63 will be a reasonable
> > > > possibility within the next few years. As the lifespan of a PG release
> > > > is about 5 years, it doesn't seem impossible that there will be actual
> > > > clusters that are going to hit this naturally in the lifespan of PG15.
> > > >
> > > > It is also possible that someone fat-fingers pg_resetwal; and creates
> > > > a cluster with LSN >= 2^63; resulting in negative values in the
> > > > s.param3 field. Not likely, but we can force such situations; and as
> > > > such we should handle that gracefully.
> > > >
> > > > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > > > the reasoning for having this function and it's named as *checkpoint*
> > > > > when it doesn't do anything specific to the checkpoint at all?
> > > >
> > > > I hadn't thought of using the types' inout functions, but it looks
> > > > like timestamp IO functions use a formatted timestring, which won't
> > > > work with the epoch-based timestamp stored in the view.
> > > >
> > > > > Having 3 unnecessary functions that aren't useful to the users at all
> > > > > in proc.dat will simply eatup the function oids IMO. Hence, I suggest
> > > > > let's try to do without extra functions.
> > > >
> > > > I agree that (1) could be simplified, or at least fully expressed in
> > > > SQL without exposing too many internals. If we're fine with exposing
> > > > internals like flags and type layouts, then (2), and arguably (4), can
> > > > be expressed in SQL as well.
> > > >
> > > > -Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-03 15:00 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-03-07 14:45 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-08 15:00 ` Nitin Jadhav <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-03-08 15:00 UTC (permalink / raw)
To: Ashutosh Sharma <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Julien Rouhaud <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> >
> > I think the output in the kind column can be displayed as {immediate,
> > force, wait, requested, time}. By the way these are all checkpoint
> > flags so it is better to display it as checkpoint flags instead of
> > checkpoint kind as mentioned in one of my previous comments.
>
> I will update in the next patch.
The current format matches with the server log message for the
checkpoint start in LogCheckpointStart(). Just to be consistent, I
have not changed the code.
I have taken care of the rest of the comments in v5 patch for which
there was clarity.
Thanks & Regards,
Nitin Jadhav
On Mon, Mar 7, 2022 at 8:15 PM Nitin Jadhav
<[email protected]> wrote:
>
> > + <row>
> > + <entry role="catalog_table_entry"><para role="column_definition">
> > + <structfield>type</structfield> <type>text</type>
> > + </para>
> > + <para>
> > + Type of checkpoint. See <xref linkend="checkpoint-types"/>.
> > + </para></entry>
> > + </row>
> > +
> > + <row>
> > + <entry role="catalog_table_entry"><para role="column_definition">
> > + <structfield>kind</structfield> <type>text</type>
> > + </para>
> > + <para>
> > + Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
> > + </para></entry>
> > + </row>
> >
> > This looks a bit confusing. Two columns, one with the name "checkpoint
> > types" and another "checkpoint kinds". You can probably rename
> > checkpoint-kinds to checkpoint-flags and let the checkpoint-types be
> > as-it-is.
>
> Makes sense. I will change in the next patch.
> ---
>
> > + <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
> > + <entry>One row only, showing the progress of the checkpoint.
> >
> > Let's make this message consistent with the already existing message
> > for pg_stat_wal_receiver. See description for pg_stat_wal_receiver
> > view in "Dynamic Statistics Views" table.
>
> You want me to change "One row only" to "Only one row" ? If that is
> the case then for other views in the "Collected Statistics Views"
> table, it is referred as "One row only". Let me know if you are
> pointing out something else.
> ---
>
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> >
> > I think the output in the kind column can be displayed as {immediate,
> > force, wait, requested, time}. By the way these are all checkpoint
> > flags so it is better to display it as checkpoint flags instead of
> > checkpoint kind as mentioned in one of my previous comments.
>
> I will update in the next patch.
> ---
>
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> > start_lsn | 0/14C60F8
> > start_time | 2022-03-03 18:59:56.018662+05:30
> > phase | performing two phase checkpoint
> >
> > This is the output I see when the checkpointer process has come out of
> > the two phase checkpoint and is currently writing checkpoint xlog
> > records and doing other stuff like updating control files etc. Is this
> > okay?
>
> The idea behind choosing the phases is based on the functionality
> which takes longer time to execute. Since after two phase checkpoint
> till post checkpoint cleanup won't take much time to execute, I have
> not added any additional phase for that. But I also agree that this
> gives wrong information to the user. How about mentioning the phase
> information at the end of each phase like "Initializing",
> "Initialization done", ..., "two phase checkpoint done", "post
> checkpoint cleanup done", .., "finalizing". Except for the first phase
> ("initializing") and last phase ("finalizing"), all the other phases
> describe the end of a certain operation. I feel this gives correct
> information even though the phase name/description does not represent
> the entire code block between two phases. For example if the current
> phase is ''two phase checkpoint done". Then the user can infer that
> the checkpointer has done till two phase checkpoint and it is doing
> other stuff that are after that. Thoughts?
> ---
>
> > The output of log_checkpoint shows the number of buffers written is 3
> > whereas the output of pg_stat_progress_checkpoint shows it as 0. See
> > below:
> >
> > 2022-03-03 20:04:45.643 IST [22043] LOG: checkpoint complete: wrote 3
> > buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled;
> > write=24.652 s, sync=104.256 s, total=3889.625 s; sync files=2,
> > longest=0.011 s, average=0.008 s; distance=0 kB, estimate=0 kB
> >
> > --
> >
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> > start_lsn | 0/14C60F8
> > start_time | 2022-03-03 18:59:56.018662+05:30
> > phase | finalizing
> > buffers_total | 0
> > buffers_processed | 0
> > buffers_written | 0
> >
> > Any idea why this mismatch?
>
> Good catch. In BufferSync() we have 'num_to_scan' (buffers_total)
> which indicates the total number of buffers to be processed. Based on
> that, the 'buffers_processed' and 'buffers_written' counter gets
> incremented. I meant these values may reach upto 'buffers_total'. The
> current pg_stat_progress_view support above information. There is
> another place when 'ckpt_bufs_written' gets incremented (In
> SlruInternalWritePage()). This increment is above the 'buffers_total'
> value and it is included in the server log message (checkpoint end)
> and not included in the view. I am a bit confused here. If we include
> this increment in the view then we cannot calculate the exact
> 'buffers_total' beforehand. Can we increment the 'buffers_toal' also
> when 'ckpt_bufs_written' gets incremented so that we can match the
> behaviour with checkpoint end message? Please share your thoughts.
> ---
>
> > I think we can add a couple of more information to this view -
> > start_time for buffer write operation and start_time for buffer sync
> > operation. These are two very time consuming tasks in a checkpoint and
> > people would find it useful to know how much time is being taken by
> > the checkpoint in I/O operation phase. thoughts?
>
> The detailed progress is getting shown for these 2 phases of the
> checkpoint like 'buffers_processed', 'buffers_written' and
> 'files_synced'. Hence I did not think about adding start time and If
> it is really required, then I can add.
>
> Thanks & Regards,
> Nitin Jadhav
>
> On Thu, Mar 3, 2022 at 8:30 PM Ashutosh Sharma <[email protected]> wrote:
> >
> > Here are some of my review comments on the latest patch:
> >
> > + <row>
> > + <entry role="catalog_table_entry"><para role="column_definition">
> > + <structfield>type</structfield> <type>text</type>
> > + </para>
> > + <para>
> > + Type of checkpoint. See <xref linkend="checkpoint-types"/>.
> > + </para></entry>
> > + </row>
> > +
> > + <row>
> > + <entry role="catalog_table_entry"><para role="column_definition">
> > + <structfield>kind</structfield> <type>text</type>
> > + </para>
> > + <para>
> > + Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
> > + </para></entry>
> > + </row>
> >
> > This looks a bit confusing. Two columns, one with the name "checkpoint
> > types" and another "checkpoint kinds". You can probably rename
> > checkpoint-kinds to checkpoint-flags and let the checkpoint-types be
> > as-it-is.
> >
> > ==
> >
> > + <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
> > + <entry>One row only, showing the progress of the checkpoint.
> >
> > Let's make this message consistent with the already existing message
> > for pg_stat_wal_receiver. See description for pg_stat_wal_receiver
> > view in "Dynamic Statistics Views" table.
> >
> > ==
> >
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> >
> > I think the output in the kind column can be displayed as {immediate,
> > force, wait, requested, time}. By the way these are all checkpoint
> > flags so it is better to display it as checkpoint flags instead of
> > checkpoint kind as mentioned in one of my previous comments.
> >
> > ==
> >
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> > start_lsn | 0/14C60F8
> > start_time | 2022-03-03 18:59:56.018662+05:30
> > phase | performing two phase checkpoint
> >
> >
> > This is the output I see when the checkpointer process has come out of
> > the two phase checkpoint and is currently writing checkpoint xlog
> > records and doing other stuff like updating control files etc. Is this
> > okay?
> >
> > ==
> >
> > The output of log_checkpoint shows the number of buffers written is 3
> > whereas the output of pg_stat_progress_checkpoint shows it as 0. See
> > below:
> >
> > 2022-03-03 20:04:45.643 IST [22043] LOG: checkpoint complete: wrote 3
> > buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled;
> > write=24.652 s, sync=104.256 s, total=3889.625 s; sync files=2,
> > longest=0.011 s, average=0.008 s; distance=0 kB, estimate=0 kB
> >
> > --
> >
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> > start_lsn | 0/14C60F8
> > start_time | 2022-03-03 18:59:56.018662+05:30
> > phase | finalizing
> > buffers_total | 0
> > buffers_processed | 0
> > buffers_written | 0
> >
> > Any idea why this mismatch?
> >
> > ==
> >
> > I think we can add a couple of more information to this view -
> > start_time for buffer write operation and start_time for buffer sync
> > operation. These are two very time consuming tasks in a checkpoint and
> > people would find it useful to know how much time is being taken by
> > the checkpoint in I/O operation phase. thoughts?
> >
> > --
> > With Regards,
> > Ashutosh Sharma.
> >
> > On Wed, Mar 2, 2022 at 4:45 PM Nitin Jadhav
> > <[email protected]> wrote:
> > >
> > > Thanks for reviewing.
> > >
> > > > > > I suggested upthread to store the starting timeline instead. This way you can
> > > > > > deduce whether it's a restartpoint or a checkpoint, but you can also deduce
> > > > > > other information, like what was the starting WAL.
> > > > >
> > > > > I don't understand why we need the timeline here to just determine
> > > > > whether it's a restartpoint or checkpoint.
> > > >
> > > > I'm not saying it's necessary, I'm saying that for the same space usage we can
> > > > store something a bit more useful. If no one cares about having the starting
> > > > timeline available for no extra cost then sure, let's just store the kind
> > > > directly.
> > >
> > > Fixed.
> > >
> > > > 2) Can't we just have these checks inside CASE-WHEN-THEN-ELSE blocks
> > > > directly instead of new function pg_stat_get_progress_checkpoint_kind?
> > > > + snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
> > > > + (flags == 0) ? "unknown" : "",
> > > > + (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
> > > > + (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
> > > > + (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
> > > > + (flags & CHECKPOINT_FORCE) ? "force " : "",
> > > > + (flags & CHECKPOINT_WAIT) ? "wait " : "",
> > > > + (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
> > > > + (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
> > > > + (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
> > >
> > > Fixed.
> > > ---
> > >
> > > > 5) Do we need a special phase for this checkpoint operation? I'm not
> > > > sure in which cases it will take a long time, but it looks like
> > > > there's a wait loop here.
> > > > vxids = GetVirtualXIDsDelayingChkpt(&nvxids);
> > > > if (nvxids > 0)
> > > > {
> > > > do
> > > > {
> > > > pg_usleep(10000L); /* wait for 10 msec */
> > > > } while (HaveVirtualXIDsDelayingChkpt(vxids, nvxids));
> > > > }
> > >
> > > Yes. It is better to add a separate phase here.
> > > ---
> > >
> > > > Also, how about special phases for SyncPostCheckpoint(),
> > > > SyncPreCheckpoint(), InvalidateObsoleteReplicationSlots(),
> > > > PreallocXlogFiles() (it currently pre-allocates only 1 WAL file, but
> > > > it might be increase in future (?)), TruncateSUBTRANS()?
> > >
> > > SyncPreCheckpoint() is just incrementing a counter and
> > > PreallocXlogFiles() currently pre-allocates only 1 WAL file. I feel
> > > there is no need to add any phases for these as of now. We can add in
> > > the future if necessary. Added phases for SyncPostCheckpoint(),
> > > InvalidateObsoleteReplicationSlots() and TruncateSUBTRANS().
> > > ---
> > >
> > > > 6) SLRU (Simple LRU) isn't a phase here, you can just say
> > > > PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES.
> > > > +
> > > > + pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
> > > > + PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
> > > > CheckPointPredicate();
> > > >
> > > > And :s/checkpointing SLRU pages/checkpointing predicate lock pages
> > > >+ WHEN 9 THEN 'checkpointing SLRU pages'
> > >
> > > Fixed.
> > > ---
> > >
> > > > 7) :s/PROGRESS_CHECKPOINT_PHASE_FILE_SYNC/PROGRESS_CHECKPOINT_PHASE_PROCESS_FILE_SYNC_REQUESTS
> > >
> > > I feel PROGRESS_CHECKPOINT_PHASE_FILE_SYNC is a better option here as
> > > it describes the purpose in less words.
> > >
> > > > And :s/WHEN 11 THEN 'performing sync requests'/WHEN 11 THEN
> > > > 'processing file sync requests'
> > >
> > > Fixed.
> > > ---
> > >
> > > > 8) :s/Finalizing/finalizing
> > > > + WHEN 14 THEN 'Finalizing'
> > >
> > > Fixed.
> > > ---
> > >
> > > > 9) :s/checkpointing snapshots/checkpointing logical replication snapshot files
> > > > + WHEN 3 THEN 'checkpointing snapshots'
> > > > :s/checkpointing logical rewrite mappings/checkpointing logical
> > > > replication rewrite mapping files
> > > > + WHEN 4 THEN 'checkpointing logical rewrite mappings'
> > >
> > > Fixed.
> > > ---
> > >
> > > > 10) I'm not sure if it's discussed, how about adding the number of
> > > > snapshot/mapping files so far the checkpoint has processed in file
> > > > processing while loops of
> > > > CheckPointSnapBuild/CheckPointLogicalRewriteHeap? Sometimes, there can
> > > > be many logical snapshot or mapping files and users may be interested
> > > > in knowing the so-far-processed-file-count.
> > >
> > > I had thought about this while sharing the v1 patch and mentioned my
> > > views upthread. I feel it won't give meaningful progress information
> > > (It can be treated as statistics). Hence not included. Thoughts?
> > >
> > > > > > As mentioned upthread, there can be multiple backends that request a
> > > > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > > > of backend that are waiting for a new checkpoint.
> > > > >
> > > > > Yeah, you are right. Let's not go that path and store an array of
> > > > > pids. I don't see a strong use-case with the pid of the process
> > > > > requesting checkpoint. If required, we can add it later once the
> > > > > pg_stat_progress_checkpoint view gets in.
> > > >
> > > > I don't think that's really necessary to give the pid list.
> > > >
> > > > If you requested a new checkpoint, it doesn't matter if it's only your backend
> > > > that triggered it, another backend or a few other dozen, the result will be the
> > > > same and you have the information that the request has been seen. We could
> > > > store just a bool for that but having a number instead also gives a bit more
> > > > information and may allow you to detect some broken logic on your client code
> > > > if it keeps increasing.
> > >
> > > It's a good metric to show in the view but the information is not
> > > readily available. Additional code is required to calculate the number
> > > of requests. Is it worth doing that? I feel this can be added later if
> > > required.
> > >
> > > Please find the v4 patch attached and share your thoughts.
> > >
> > > Thanks & Regards,
> > > Nitin Jadhav
> > >
> > > On Tue, Mar 1, 2022 at 2:27 PM Nitin Jadhav
> > > <[email protected]> wrote:
> > > >
> > > > > > 3) Why do we need this extra calculation for start_lsn?
> > > > > > Do you ever see a negative LSN or something here?
> > > > > > + ('0/0'::pg_lsn + (
> > > > > > + CASE
> > > > > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > > > > + ELSE (0)::numeric
> > > > > > + END + (s.param3)::numeric)) AS start_lsn,
> > > > >
> > > > > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > > > > bigint type; thus the signed int64. This cast is OK as it wraps
> > > > > around, but that means we have to take care to correctly display the
> > > > > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > > > > the special-casing for negative values.
> > > >
> > > > Yes. The extra calculation is required here as we are storing unit64
> > > > value in the variable of type int64. When we convert uint64 to int64
> > > > then the bit pattern is preserved (so no data is lost). The high-order
> > > > bit becomes the sign bit and if the sign bit is set, both the sign and
> > > > magnitude of the value changes. To safely get the actual uint64 value
> > > > whatever was assigned, we need the above calculations.
> > > >
> > > > > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > > > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > > > > the reasoning for having this function and it's named as *checkpoint*
> > > > > > when it doesn't do anything specific to the checkpoint at all?
> > > > >
> > > > > I hadn't thought of using the types' inout functions, but it looks
> > > > > like timestamp IO functions use a formatted timestring, which won't
> > > > > work with the epoch-based timestamp stored in the view.
> > > >
> > > > There is a variation of to_timestamp() which takes UNIX epoch (float8)
> > > > as an argument and converts it to timestamptz but we cannot directly
> > > > call this function with S.param4.
> > > >
> > > > TimestampTz
> > > > GetCurrentTimestamp(void)
> > > > {
> > > > TimestampTz result;
> > > > struct timeval tp;
> > > >
> > > > gettimeofday(&tp, NULL);
> > > >
> > > > result = (TimestampTz) tp.tv_sec -
> > > > ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY);
> > > > result = (result * USECS_PER_SEC) + tp.tv_usec;
> > > >
> > > > return result;
> > > > }
> > > >
> > > > S.param4 contains the output of the above function
> > > > (GetCurrentTimestamp()) which returns Postgres epoch but the
> > > > to_timestamp() expects UNIX epoch as input. So some calculation is
> > > > required here. I feel the SQL 'to_timestamp(946684800 +
> > > > (S.param4::float / 1000000)) AS start_time' works fine. The value
> > > > '946684800' is equal to ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) *
> > > > SECS_PER_DAY). I am not sure whether it is good practice to use this
> > > > way. Kindly share your thoughts.
> > > >
> > > > Thanks & Regards,
> > > > Nitin Jadhav
> > > >
> > > > On Mon, Feb 28, 2022 at 6:40 PM Matthias van de Meent
> > > > <[email protected]> wrote:
> > > > >
> > > > > On Sun, 27 Feb 2022 at 16:14, Bharath Rupireddy
> > > > > <[email protected]> wrote:
> > > > > > 3) Why do we need this extra calculation for start_lsn?
> > > > > > Do you ever see a negative LSN or something here?
> > > > > > + ('0/0'::pg_lsn + (
> > > > > > + CASE
> > > > > > + WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
> > > > > > + ELSE (0)::numeric
> > > > > > + END + (s.param3)::numeric)) AS start_lsn,
> > > > >
> > > > > Yes: LSN can take up all of an uint64; whereas the pgstat column is a
> > > > > bigint type; thus the signed int64. This cast is OK as it wraps
> > > > > around, but that means we have to take care to correctly display the
> > > > > LSN when it is > 0x7FFF_FFFF_FFFF_FFFF; which is what we do here using
> > > > > the special-casing for negative values.
> > > > >
> > > > > As to whether it is reasonable: Generating 16GB of wal every second
> > > > > (2^34 bytes /sec) is probably not impossible (cpu <> memory bandwidth
> > > > > has been > 20GB/sec for a while); and that leaves you 2^29 seconds of
> > > > > database runtime; or about 17 years. Seeing that a cluster can be
> > > > > `pg_upgrade`d (which doesn't reset cluster LSN) since PG 9.0 from at
> > > > > least version PG 8.4.0 (2009) (and through pg_migrator, from 8.3.0)),
> > > > > we can assume that clusters hitting LSN=2^63 will be a reasonable
> > > > > possibility within the next few years. As the lifespan of a PG release
> > > > > is about 5 years, it doesn't seem impossible that there will be actual
> > > > > clusters that are going to hit this naturally in the lifespan of PG15.
> > > > >
> > > > > It is also possible that someone fat-fingers pg_resetwal; and creates
> > > > > a cluster with LSN >= 2^63; resulting in negative values in the
> > > > > s.param3 field. Not likely, but we can force such situations; and as
> > > > > such we should handle that gracefully.
> > > > >
> > > > > > 4) Can't you use timestamptz_in(to_char(s.param4)) instead of
> > > > > > pg_stat_get_progress_checkpoint_start_time? I don't quite understand
> > > > > > the reasoning for having this function and it's named as *checkpoint*
> > > > > > when it doesn't do anything specific to the checkpoint at all?
> > > > >
> > > > > I hadn't thought of using the types' inout functions, but it looks
> > > > > like timestamp IO functions use a formatted timestring, which won't
> > > > > work with the epoch-based timestamp stored in the view.
> > > > >
> > > > > > Having 3 unnecessary functions that aren't useful to the users at all
> > > > > > in proc.dat will simply eatup the function oids IMO. Hence, I suggest
> > > > > > let's try to do without extra functions.
> > > > >
> > > > > I agree that (1) could be simplified, or at least fully expressed in
> > > > > SQL without exposing too many internals. If we're fine with exposing
> > > > > internals like flags and type layouts, then (2), and arguably (4), can
> > > > > be expressed in SQL as well.
> > > > >
> > > > > -Matthias
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-03 18:28 ` Julien Rouhaud <[email protected]>
2022-03-04 11:29 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-06-06 06:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2 siblings, 2 replies; 85+ messages in thread
From: Julien Rouhaud @ 2022-03-03 18:28 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Ashutosh Sharma <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Mar 2, 2022 at 7:15 PM Nitin Jadhav
<[email protected]> wrote:
>
> > > > As mentioned upthread, there can be multiple backends that request a
> > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > of backend that are waiting for a new checkpoint.
>
> It's a good metric to show in the view but the information is not
> readily available. Additional code is required to calculate the number
> of requests. Is it worth doing that? I feel this can be added later if
> required.
Is it that hard or costly to do? Just sending a message to increment
the stat counter in RequestCheckpoint() would be enough.
Also, unless I'm missing something it's still only showing the initial
checkpoint flags, so it's *not* showing what the checkpoint is really
doing, only what the checkpoint may be doing if nothing else happens.
It just feels wrong. You could even use that ckpt_flags info to know
that at least one backend has requested a new checkpoint, if you don't
want to have a number of backends.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-03 18:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-03-04 11:29 ` Nitin Jadhav <[email protected]>
2022-03-04 12:20 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
1 sibling, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-03-04 11:29 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Ashutosh Sharma <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Thanks for reviewing.
> 6) How about shutdown and end-of-recovery checkpoint? Are you planning
> to have an ereport_startup_progress mechanism as 0002?
I thought of including it earlier then I felt lets first make the
current patch stable. Once all the fields are properly decided and the
patch gets in then we can easily extend the functionality to shutdown
and end-of-recovery cases. I have also observed that the timer
functionality wont work properly in case of shutdown as we are doing
an immediate checkpoint. So this needs a lot of discussion and I would
like to handle this on a separate thread.
---
> 7) I think you don't need to call checkpoint_progress_start and
> pgstat_progress_update_param, any other progress reporting function
> for shutdown and end-of-recovery checkpoint right?
I had included the guards earlier and then removed later based on the
discussion upthread.
---
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
> start_lsn | 0/14C60F8
> start_time | 2022-03-03 18:59:56.018662+05:30
> phase | performing two phase checkpoint
>
>
> This is the output I see when the checkpointer process has come out of
> the two phase checkpoint and is currently writing checkpoint xlog
> records and doing other stuff like updating control files etc. Is this
> okay?
The idea behind choosing the phases is based on the functionality
which takes longer time to execute. Since after two phase checkpoint
till post checkpoint cleanup won't take much time to execute, I have
not added any additional phase for that. But I also agree that this
gives wrong information to the user. How about mentioning the phase
information at the end of each phase like "Initializing",
"Initialization done", ..., "two phase checkpoint done", "post
checkpoint cleanup done", .., "finalizing". Except for the first phase
("initializing") and last phase ("finalizing"), all the other phases
describe the end of a certain operation. I feel this gives correct
information even though the phase name/description does not represent
the entire code block between two phases. For example if the current
phase is ''two phase checkpoint done". Then the user can infer that
the checkpointer has done till two phase checkpoint and it is doing
other stuff that are after that. Thoughts?
> The output of log_checkpoint shows the number of buffers written is 3
> whereas the output of pg_stat_progress_checkpoint shows it as 0. See
> below:
>
> 2022-03-03 20:04:45.643 IST [22043] LOG: checkpoint complete: wrote 3
> buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled;
> write=24.652 s, sync=104.256 s, total=3889.625 s; sync files=2,
> longest=0.011 s, average=0.008 s; distance=0 kB, estimate=0 kB
>
> --
>
> [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> -[ RECORD 1 ]-----+-------------------------------------
> pid | 22043
> type | checkpoint
> kind | immediate force wait requested time
> start_lsn | 0/14C60F8
> start_time | 2022-03-03 18:59:56.018662+05:30
> phase | finalizing
> buffers_total | 0
> buffers_processed | 0
> buffers_written | 0
>
> Any idea why this mismatch?
Good catch. In BufferSync() we have 'num_to_scan' (buffers_total)
which indicates the total number of buffers to be processed. Based on
that, the 'buffers_processed' and 'buffers_written' counter gets
incremented. I meant these values may reach upto 'buffers_total'. The
current pg_stat_progress_view support above information. There is
another place when 'ckpt_bufs_written' gets incremented (In
SlruInternalWritePage()). This increment is above the 'buffers_total'
value and it is included in the server log message (checkpoint end)
and not included in the view. I am a bit confused here. If we include
this increment in the view then we cannot calculate the exact
'buffers_total' beforehand. Can we increment the 'buffers_toal' also
when 'ckpt_bufs_written' gets incremented so that we can match the
behaviour with checkpoint end message? Please share your thoughts.
---
> I think we can add a couple of more information to this view -
> start_time for buffer write operation and start_time for buffer sync
> operation. These are two very time consuming tasks in a checkpoint and
> people would find it useful to know how much time is being taken by
> the checkpoint in I/O operation phase. thoughts?
I felt the detailed progress is getting shown for these 2 phases of
the checkpoint like 'buffers_processed', 'buffers_written' and
'files_synced'. Hence I did not think about adding start time and If
it is really required, then I can add.
> Is it that hard or costly to do? Just sending a message to increment
> the stat counter in RequestCheckpoint() would be enough.
>
> Also, unless I'm missing something it's still only showing the initial
> checkpoint flags, so it's *not* showing what the checkpoint is really
> doing, only what the checkpoint may be doing if nothing else happens.
> It just feels wrong. You could even use that ckpt_flags info to know
> that at least one backend has requested a new checkpoint, if you don't
> want to have a number of backends.
I think using ckpt_flags to display whether any new requests have been
made or not is a good idea. I will include it in the next patch.
Thanks & Regards,
Nitin Jadhav
On Thu, Mar 3, 2022 at 11:58 PM Julien Rouhaud <[email protected]> wrote:
>
> On Wed, Mar 2, 2022 at 7:15 PM Nitin Jadhav
> <[email protected]> wrote:
> >
> > > > > As mentioned upthread, there can be multiple backends that request a
> > > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > > of backend that are waiting for a new checkpoint.
> >
> > It's a good metric to show in the view but the information is not
> > readily available. Additional code is required to calculate the number
> > of requests. Is it worth doing that? I feel this can be added later if
> > required.
>
> Is it that hard or costly to do? Just sending a message to increment
> the stat counter in RequestCheckpoint() would be enough.
>
> Also, unless I'm missing something it's still only showing the initial
> checkpoint flags, so it's *not* showing what the checkpoint is really
> doing, only what the checkpoint may be doing if nothing else happens.
> It just feels wrong. You could even use that ckpt_flags info to know
> that at least one backend has requested a new checkpoint, if you don't
> want to have a number of backends.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-03 18:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-03-04 11:29 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-03-04 12:20 ` Ashutosh Sharma <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Ashutosh Sharma @ 2022-03-04 12:20 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Please don't mix comments from multiple reviewers into one thread.
It's hard to understand which comments are mine or Julien's or from
others. Can you please respond to the email from each of us separately
with an inline response. That will be helpful to understand your
thoughts on our review comments.
--
With Regards,
Ashutosh Sharma.
On Fri, Mar 4, 2022 at 4:59 PM Nitin Jadhav
<[email protected]> wrote:
>
> Thanks for reviewing.
>
> > 6) How about shutdown and end-of-recovery checkpoint? Are you planning
> > to have an ereport_startup_progress mechanism as 0002?
>
> I thought of including it earlier then I felt lets first make the
> current patch stable. Once all the fields are properly decided and the
> patch gets in then we can easily extend the functionality to shutdown
> and end-of-recovery cases. I have also observed that the timer
> functionality wont work properly in case of shutdown as we are doing
> an immediate checkpoint. So this needs a lot of discussion and I would
> like to handle this on a separate thread.
> ---
>
> > 7) I think you don't need to call checkpoint_progress_start and
> > pgstat_progress_update_param, any other progress reporting function
> > for shutdown and end-of-recovery checkpoint right?
>
> I had included the guards earlier and then removed later based on the
> discussion upthread.
> ---
>
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> > start_lsn | 0/14C60F8
> > start_time | 2022-03-03 18:59:56.018662+05:30
> > phase | performing two phase checkpoint
> >
> >
> > This is the output I see when the checkpointer process has come out of
> > the two phase checkpoint and is currently writing checkpoint xlog
> > records and doing other stuff like updating control files etc. Is this
> > okay?
>
> The idea behind choosing the phases is based on the functionality
> which takes longer time to execute. Since after two phase checkpoint
> till post checkpoint cleanup won't take much time to execute, I have
> not added any additional phase for that. But I also agree that this
> gives wrong information to the user. How about mentioning the phase
> information at the end of each phase like "Initializing",
> "Initialization done", ..., "two phase checkpoint done", "post
> checkpoint cleanup done", .., "finalizing". Except for the first phase
> ("initializing") and last phase ("finalizing"), all the other phases
> describe the end of a certain operation. I feel this gives correct
> information even though the phase name/description does not represent
> the entire code block between two phases. For example if the current
> phase is ''two phase checkpoint done". Then the user can infer that
> the checkpointer has done till two phase checkpoint and it is doing
> other stuff that are after that. Thoughts?
>
> > The output of log_checkpoint shows the number of buffers written is 3
> > whereas the output of pg_stat_progress_checkpoint shows it as 0. See
> > below:
> >
> > 2022-03-03 20:04:45.643 IST [22043] LOG: checkpoint complete: wrote 3
> > buffers (0.0%); 0 WAL file(s) added, 0 removed, 0 recycled;
> > write=24.652 s, sync=104.256 s, total=3889.625 s; sync files=2,
> > longest=0.011 s, average=0.008 s; distance=0 kB, estimate=0 kB
> >
> > --
> >
> > [local]:5432 ashu@postgres=# select * from pg_stat_progress_checkpoint;
> > -[ RECORD 1 ]-----+-------------------------------------
> > pid | 22043
> > type | checkpoint
> > kind | immediate force wait requested time
> > start_lsn | 0/14C60F8
> > start_time | 2022-03-03 18:59:56.018662+05:30
> > phase | finalizing
> > buffers_total | 0
> > buffers_processed | 0
> > buffers_written | 0
> >
> > Any idea why this mismatch?
>
> Good catch. In BufferSync() we have 'num_to_scan' (buffers_total)
> which indicates the total number of buffers to be processed. Based on
> that, the 'buffers_processed' and 'buffers_written' counter gets
> incremented. I meant these values may reach upto 'buffers_total'. The
> current pg_stat_progress_view support above information. There is
> another place when 'ckpt_bufs_written' gets incremented (In
> SlruInternalWritePage()). This increment is above the 'buffers_total'
> value and it is included in the server log message (checkpoint end)
> and not included in the view. I am a bit confused here. If we include
> this increment in the view then we cannot calculate the exact
> 'buffers_total' beforehand. Can we increment the 'buffers_toal' also
> when 'ckpt_bufs_written' gets incremented so that we can match the
> behaviour with checkpoint end message? Please share your thoughts.
> ---
>
> > I think we can add a couple of more information to this view -
> > start_time for buffer write operation and start_time for buffer sync
> > operation. These are two very time consuming tasks in a checkpoint and
> > people would find it useful to know how much time is being taken by
> > the checkpoint in I/O operation phase. thoughts?
>
> I felt the detailed progress is getting shown for these 2 phases of
> the checkpoint like 'buffers_processed', 'buffers_written' and
> 'files_synced'. Hence I did not think about adding start time and If
> it is really required, then I can add.
>
> > Is it that hard or costly to do? Just sending a message to increment
> > the stat counter in RequestCheckpoint() would be enough.
> >
> > Also, unless I'm missing something it's still only showing the initial
> > checkpoint flags, so it's *not* showing what the checkpoint is really
> > doing, only what the checkpoint may be doing if nothing else happens.
> > It just feels wrong. You could even use that ckpt_flags info to know
> > that at least one backend has requested a new checkpoint, if you don't
> > want to have a number of backends.
>
> I think using ckpt_flags to display whether any new requests have been
> made or not is a good idea. I will include it in the next patch.
>
> Thanks & Regards,
> Nitin Jadhav
> On Thu, Mar 3, 2022 at 11:58 PM Julien Rouhaud <[email protected]> wrote:
> >
> > On Wed, Mar 2, 2022 at 7:15 PM Nitin Jadhav
> > <[email protected]> wrote:
> > >
> > > > > > As mentioned upthread, there can be multiple backends that request a
> > > > > > checkpoint, so unless we want to store an array of pid we should store a number
> > > > > > of backend that are waiting for a new checkpoint.
> > >
> > > It's a good metric to show in the view but the information is not
> > > readily available. Additional code is required to calculate the number
> > > of requests. Is it worth doing that? I feel this can be added later if
> > > required.
> >
> > Is it that hard or costly to do? Just sending a message to increment
> > the stat counter in RequestCheckpoint() would be enough.
> >
> > Also, unless I'm missing something it's still only showing the initial
> > checkpoint flags, so it's *not* showing what the checkpoint is really
> > doing, only what the checkpoint may be doing if nothing else happens.
> > It just feels wrong. You could even use that ckpt_flags info to know
> > that at least one backend has requested a new checkpoint, if you don't
> > want to have a number of backends.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-03 18:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-06-06 06:03 ` Nitin Jadhav <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-06-06 06:03 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Ashutosh Sharma <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
Here is the update patch which fixes the previous comments discussed
in this thread. I am sorry for the long gap in the discussion. Kindly
let me know if I have missed any of the comments or anything new.
Thanks & Regards,
Nitin Jadhav
On Fri, Mar 18, 2022 at 4:52 PM Nitin Jadhav
<[email protected]> wrote:
>
> > > > I don't get it. The checkpoint flags and the view flags (set by
> > > > pgstat_progrss_update*) are different, so why can't we add this flag to the
> > > > view flags? The fact that checkpointer.c doesn't update the passed flag and
> > > > instead look in the shmem to see if CHECKPOINT_IMMEDIATE has been set since is
> > > > an implementation detail, and the view shouldn't focus on which flags were
> > > > initially passed to the checkpointer but instead which flags the checkpointer
> > > > is actually enforcing, as that's what the user should be interested in. If you
> > > > want to store it in another field internally but display it in the view with
> > > > the rest of the flags, I'm fine with it.
> > >
> > > Just to be in sync with the way code behaves, it is better not to
> > > update the next checkpoint request's CHECKPOINT_IMMEDIATE with the
> > > current checkpoint 'flags' field. Because the current checkpoint
> > > starts with a different set of flags and when there is a new request
> > > (with CHECKPOINT_IMMEDIATE), it just processes the pending operations
> > > quickly to take up next requests. If we update this information in the
> > > 'flags' field of the view, it says that the current checkpoint is
> > > started with CHECKPOINT_IMMEDIATE which is not true.
> >
> > Which is why I suggested to only take into account CHECKPOINT_REQUESTED (to
> > be able to display that a new checkpoint was requested)
>
> I will take care in the next patch.
>
> > > Hence I had
> > > thought of adding a new field ('next flags' or 'upcoming flags') which
> > > contain all the flag values of new checkpoint requests. This field
> > > indicates whether the current checkpoint is throttled or not and also
> > > it indicates there are new requests.
> >
> > I'm not opposed to having such a field, I'm opposed to having a view with "the
> > current checkpoint is throttled but if there are some flags in the next
> > checkpoint flags and those flags contain checkpoint immediate then the current
> > checkpoint isn't actually throttled anymore" behavior.
>
> I understand your point and I also agree that it becomes difficult for
> the user to understand the context.
>
> > and
> > CHECKPOINT_IMMEDIATE, to be able to display that the current checkpoint isn't
> > throttled anymore if it were.
> >
> > I still don't understand why you want so much to display "how the checkpoint
> > was initially started" rather than "how the checkpoint is really behaving right
> > now". The whole point of having a progress view is to have something dynamic
> > that reflects the current activity.
>
> As of now I will not consider adding this information to the view. If
> required and nobody opposes having this included in the 'flags' field
> of the view, then I will consider adding.
>
> Thanks & Regards,
> Nitin Jadhav
>
> On Mon, Mar 14, 2022 at 5:16 PM Julien Rouhaud <[email protected]> wrote:
> >
> > On Mon, Mar 14, 2022 at 03:16:50PM +0530, Nitin Jadhav wrote:
> > > > > I am not suggesting
> > > > > removing the existing 'flags' field of pg_stat_progress_checkpoint
> > > > > view and adding a new field 'throttled'. The content of the 'flags'
> > > > > field remains the same. I was suggesting replacing the 'next_flags'
> > > > > field with 'throttled' field since the new request with
> > > > > CHECKPOINT_IMMEDIATE flag enabled will affect the current checkpoint.
> > > >
> > > > Are you saying that this new throttled flag will only be set by the overloaded
> > > > flags in ckpt_flags?
> > >
> > > Yes. you are right.
> > >
> > > > So you can have a checkpoint with a CHECKPOINT_IMMEDIATE
> > > > flags that's throttled, and a checkpoint without the CHECKPOINT_IMMEDIATE flag
> > > > that's not throttled?
> > >
> > > I think it's the reverse. A checkpoint with a CHECKPOINT_IMMEDIATE
> > > flags that's not throttled (disables delays between writes) and a
> > > checkpoint without the CHECKPOINT_IMMEDIATE flag that's throttled
> > > (enables delays between writes)
> >
> > Yes that's how it's supposed to work, but my point was that your suggested
> > 'throttled' flag could say the opposite, which is bad.
> >
> > > > I don't get it. The checkpoint flags and the view flags (set by
> > > > pgstat_progrss_update*) are different, so why can't we add this flag to the
> > > > view flags? The fact that checkpointer.c doesn't update the passed flag and
> > > > instead look in the shmem to see if CHECKPOINT_IMMEDIATE has been set since is
> > > > an implementation detail, and the view shouldn't focus on which flags were
> > > > initially passed to the checkpointer but instead which flags the checkpointer
> > > > is actually enforcing, as that's what the user should be interested in. If you
> > > > want to store it in another field internally but display it in the view with
> > > > the rest of the flags, I'm fine with it.
> > >
> > > Just to be in sync with the way code behaves, it is better not to
> > > update the next checkpoint request's CHECKPOINT_IMMEDIATE with the
> > > current checkpoint 'flags' field. Because the current checkpoint
> > > starts with a different set of flags and when there is a new request
> > > (with CHECKPOINT_IMMEDIATE), it just processes the pending operations
> > > quickly to take up next requests. If we update this information in the
> > > 'flags' field of the view, it says that the current checkpoint is
> > > started with CHECKPOINT_IMMEDIATE which is not true.
> >
> > Which is why I suggested to only take into account CHECKPOINT_REQUESTED (to
> > be able to display that a new checkpoint was requested) and
> > CHECKPOINT_IMMEDIATE, to be able to display that the current checkpoint isn't
> > throttled anymore if it were.
> >
> > I still don't understand why you want so much to display "how the checkpoint
> > was initially started" rather than "how the checkpoint is really behaving right
> > now". The whole point of having a progress view is to have something dynamic
> > that reflects the current activity.
> >
> > > Hence I had
> > > thought of adding a new field ('next flags' or 'upcoming flags') which
> > > contain all the flag values of new checkpoint requests. This field
> > > indicates whether the current checkpoint is throttled or not and also
> > > it indicates there are new requests.
> >
> > I'm not opposed to having such a field, I'm opposed to having a view with "the
> > current checkpoint is throttled but if there are some flags in the next
> > checkpoint flags and those flags contain checkpoint immediate then the current
> > checkpoint isn't actually throttled anymore" behavior.
Attachments:
[application/octet-stream] v6-0001-pg_stat_progress_checkpoint-view.patch (38.0K, ../../CAMm1aWZ9eP==mk_UcO_LkYTx9oJLd98KYBbbsxp6r9DeDPbZ4g@mail.gmail.com/2-v6-0001-pg_stat_progress_checkpoint-view.patch)
download | inline diff:
From 104184f6cba7dfe33eb710de8158b21e6937bf51 Mon Sep 17 00:00:00 2001
From: Nitin Jadhav <[email protected]>
Date: Mon, 6 Jun 2022 05:55:06 +0000
Subject: [PATCH] pg_stat_progress_checkpoint-view
---
doc/src/sgml/monitoring.sgml | 405 +++++++++++++++++++++++++-
doc/src/sgml/ref/checkpoint.sgml | 7 +
doc/src/sgml/wal.sgml | 6 +-
src/backend/access/transam/xlog.c | 102 +++++++
src/backend/catalog/system_views.sql | 51 ++++
src/backend/postmaster/checkpointer.c | 15 +-
src/backend/storage/buffer/bufmgr.c | 7 +
src/backend/storage/sync/sync.c | 6 +
src/backend/utils/adt/pgstatfuncs.c | 2 +
src/include/commands/progress.h | 38 +++
src/include/utils/backend_progress.h | 3 +-
src/test/regress/expected/rules.out | 70 +++++
12 files changed, 706 insertions(+), 6 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 4549c2560e..5fe0ba4492 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -414,6 +414,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the checkpoint.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -5736,7 +5743,7 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
which support progress reporting are <command>ANALYZE</command>,
<command>CLUSTER</command>,
<command>CREATE INDEX</command>, <command>VACUUM</command>,
- <command>COPY</command>,
+ <command>COPY</command>, <command>CHECKPOINT</command>
and <xref linkend="protocol-replication-base-backup"/> (i.e., replication
command that <xref linkend="app-pgbasebackup"/> issues to take
a base backup).
@@ -7024,6 +7031,402 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>Checkpoint Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the checkpointer process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of the checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>flags</structfield> <type>text</type>
+ </para>
+ <para>
+ Flags of the checkpoint. See <xref linkend="checkpoint-flags"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_lsn</structfield> <type>text</type>
+ </para>
+ <para>
+ The checkpoint start location.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_time</structfield> <type>timestamp with time zone</type>
+ </para>
+ <para>
+ Start time of the checkpoint.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal>buffers_total</literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>buffers_total</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>files_total</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>new_requests</structfield> <type>text</type>
+ </para>
+ <para>
+ True if any of the backend is requested for a checkpoint while the
+ current checkpoint is in progress, False otherwise.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-types">
+ <title>Checkpoint Types</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Types</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>checkpoint</literal></entry>
+ <entry>
+ The current operation is checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>restartpoint</literal></entry>
+ <entry>
+ The current operation is restartpoint.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-flags">
+ <title>Checkpoint Flags</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Flags</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>shutdown</literal></entry>
+ <entry>
+ The checkpoint is for shutdown.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>end-of-recovery</literal></entry>
+ <entry>
+ The checkpoint is for end-of-recovery.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>immediate</literal></entry>
+ <entry>
+ The checkpoint happens without delays.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint is started because some operation (for which the
+ checkpoint is necessary) forced a checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>flush all</literal></entry>
+ <entry>
+ The checkpoint flushes all pages, including those belonging to unlogged
+ tables.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wait</literal></entry>
+ <entry>
+ The operations which requested the checkpoint waits for completion
+ before returning.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>requested</literal></entry>
+ <entry>
+ The checkpoint request has been made.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>max_wal_size</literal> is
+ reached.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>checkpoint_timeout</literal>
+ expired.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>Checkpoint Phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The checkpointer process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>getting virtual transaction IDs</literal></entry>
+ <entry>
+ The checkpointer process is getting the virtual transaction IDs that
+ are delaying the checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently flushing all the replication slots
+ to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical replication snapshot files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing all the serialized
+ snapshot files that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mapping files</literal></entry>
+ <entry>
+ The checkpointer process is currently removing unwanted or flushing
+ required logical rewrite mapping files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication origin</literal></entry>
+ <entry>
+ The checkpointer process is currently performing a checkpoint of each
+ replication origin's progress with respect to the replayed remote LSN.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit log pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit log pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit time stamp pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit time stamp pages to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing subtransaction pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing subtransaction pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing multixact pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing multixact pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing predicate lock pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing predicate lock pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The checkpointer process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>processing file sync requests</literal></entry>
+ <entry>
+ The checkpointer process is currently processing file sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The checkpointer process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing post checkpoint cleanup</literal></entry>
+ <entry>
+ The checkpointer process is currently performing post checkpoint cleanup.
+ It removes any lingering files that can be safely removed.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>invalidating replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently invalidating replication slots.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old WAL files</literal></entry>
+ <entry>
+ The checkpointer process is currently recycling old WAL files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>truncating subtransactions</literal></entry>
+ <entry>
+ The checkpointer process is currently removing the subtransaction
+ segments.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The checkpointer process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/doc/src/sgml/ref/checkpoint.sgml b/doc/src/sgml/ref/checkpoint.sgml
index 1cebc03d15..f33db50cfc 100644
--- a/doc/src/sgml/ref/checkpoint.sgml
+++ b/doc/src/sgml/ref/checkpoint.sgml
@@ -56,6 +56,13 @@ CHECKPOINT
the <link linkend="predefined-roles-table"><literal>pg_checkpointer</literal></link>
role can call <command>CHECKPOINT</command>.
</para>
+
+ <para>
+ The checkpointer process running the checkpoint will report its progress
+ in the <structname>pg_stat_progress_checkpoint</structname> view except for
+ the shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
+ </para>
</refsect1>
<refsect1>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 4b6ef283c1..607f21dfd4 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -530,7 +530,11 @@
adjust the <xref linkend="guc-archive-timeout"/> parameter rather than the
checkpoint parameters.)
It is also possible to force a checkpoint by using the SQL
- command <command>CHECKPOINT</command>.
+ command <command>CHECKPOINT</command>. The checkpointer process running the
+ checkpoint will report its progress in the
+ <structname>pg_stat_progress_checkpoint</structname> view except for the
+ shutdown and end-of-recovery cases. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
</para>
<para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 71136b11a2..8272d02b1e 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -66,6 +66,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "commands/progress.h"
#include "common/controldata_utils.h"
#include "common/file_utils.h"
#include "executor/instrument.h"
@@ -695,6 +696,8 @@ static void WALInsertLockAcquireExclusive(void);
static void WALInsertLockRelease(void);
static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static void checkpoint_progress_start(int flags, int type);
+
/*
* Insert an XLOG record represented by an already-constructed chain of data
* chunks. This is a low-level routine; to construct the WAL record header
@@ -6429,6 +6432,9 @@ CreateCheckPoint(int flags)
XLogCtl->RedoRecPtr = checkPoint.redo;
SpinLockRelease(&XLogCtl->info_lck);
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_CHECKPOINT);
+
/*
* If enabled, log checkpoint start. We postpone this until now so as not
* to log anything if we decided to skip the checkpoint.
@@ -6511,6 +6517,8 @@ CreateCheckPoint(int flags)
* clog and we will correctly flush the update below. So we cannot miss
* any xacts we need to wait for.
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS);
vxids = GetVirtualXIDsDelayingChkpt(&nvxids, DELAY_CHKPT_START);
if (nvxids > 0)
{
@@ -6626,6 +6634,8 @@ CreateCheckPoint(int flags)
/*
* Let smgr do post-checkpoint cleanup (eg, deleting old files).
*/
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP);
SyncPostCheckpoint();
/*
@@ -6641,6 +6651,9 @@ CreateCheckPoint(int flags)
*/
XLByteToSeg(RedoRecPtr, _logSegNo, wal_segment_size);
KeepLogSeg(recptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -6651,6 +6664,8 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
@@ -6669,11 +6684,21 @@ CreateCheckPoint(int flags)
* StartupSUBTRANS hasn't been called yet.
*/
if (!RecoveryInProgress())
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -6830,29 +6855,63 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN);
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES);
CheckPointPredicate();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SYNC_FILES);
ProcessSyncRequests();
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -7002,6 +7061,9 @@ CreateRestartPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the restartpoint. */
+ checkpoint_progress_start(flags, PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT);
+
if (log_checkpoints)
LogCheckpointStart(flags, true);
@@ -7085,6 +7147,9 @@ CreateRestartPoint(int flags)
replayPtr = GetXLogReplayRecPtr(&replayTLI);
endptr = (receivePtr < replayPtr) ? replayPtr : receivePtr;
KeepLogSeg(endptr, &_logSegNo);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS);
+
if (InvalidateObsoleteReplicationSlots(_logSegNo))
{
/*
@@ -7111,6 +7176,8 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
/*
@@ -7127,11 +7194,20 @@ CreateRestartPoint(int flags)
* this because StartupSUBTRANS hasn't been called yet.
*/
if (EnableHotStandby)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS);
TruncateSUBTRANS(GetOldestTransactionIdConsideredRunning());
+ }
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the restartpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -8880,3 +8956,29 @@ SetWalWriterSleeping(bool sleeping)
XLogCtl->WalWriterSleeping = sleeping;
SpinLockRelease(&XLogCtl->info_lck);
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+static void
+checkpoint_progress_start(int flags, int type)
+{
+ const int index[] = {
+ PROGRESS_CHECKPOINT_TYPE,
+ PROGRESS_CHECKPOINT_FLAGS,
+ PROGRESS_CHECKPOINT_LSN,
+ PROGRESS_CHECKPOINT_START_TIMESTAMP,
+ PROGRESS_CHECKPOINT_PHASE
+ };
+ int64 val[5];
+
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+
+ val[0] = type;
+ val[1] = flags;
+ val[2] = RedoRecPtr;
+ val[3] = CheckpointStats.ckpt_start_t;
+ val[4] = PROGRESS_CHECKPOINT_PHASE_INIT;
+
+ pgstat_progress_update_multi_param(5, index, val);
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index fedaed533b..20d029a547 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1265,6 +1265,57 @@ CREATE VIEW pg_stat_progress_copy AS
FROM pg_stat_get_progress_info('COPY') AS S
LEFT JOIN pg_database D ON S.datid = D.oid;
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ CASE S.param1 WHEN 1 THEN 'checkpoint'
+ WHEN 2 THEN 'restartpoint'
+ END AS type,
+ ( CASE WHEN (S.param2 & 4) > 0 THEN 'immediate ' ELSE '' END ||
+ CASE WHEN (S.param2 & 8) > 0 THEN 'force ' ELSE '' END ||
+ CASE WHEN (S.param2 & 16) > 0 THEN 'flush-all ' ELSE '' END ||
+ CASE WHEN (S.param2 & 32) > 0 THEN 'wait ' ELSE '' END ||
+ CASE WHEN (S.param2 & 128) > 0 THEN 'wal ' ELSE '' END ||
+ CASE WHEN (S.param2 & 256) > 0 THEN 'time ' ELSE '' END
+ ) AS flags,
+ ( '0/0'::pg_lsn +
+ ((CASE
+ WHEN S.param3 < 0 THEN pow(2::numeric, 64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ S.param3::numeric)
+ ) AS start_lsn,
+ to_timestamp(946684800 + (S.param4::float8 / 1000000)) AS start_time,
+ CASE S.param5 WHEN 1 THEN 'initializing'
+ WHEN 2 THEN 'getting virtual transaction IDs'
+ WHEN 3 THEN 'checkpointing replication slots'
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'
+ WHEN 6 THEN 'checkpointing replication origin'
+ WHEN 7 THEN 'checkpointing commit log pages'
+ WHEN 8 THEN 'checkpointing commit time stamp pages'
+ WHEN 9 THEN 'checkpointing subtransaction pages'
+ WHEN 10 THEN 'checkpointing multixact pages'
+ WHEN 11 THEN 'checkpointing predicate lock pages'
+ WHEN 12 THEN 'checkpointing buffers'
+ WHEN 13 THEN 'processing file sync requests'
+ WHEN 14 THEN 'performing two phase checkpoint'
+ WHEN 15 THEN 'performing post checkpoint cleanup'
+ WHEN 16 THEN 'invalidating replication slots'
+ WHEN 17 THEN 'recycling old WAL files'
+ WHEN 18 THEN 'truncating subtransactions'
+ WHEN 19 THEN 'finalizing'
+ END AS phase,
+ S.param6 AS buffers_total,
+ S.param7 AS buffers_processed,
+ S.param8 AS buffers_written,
+ S.param9 AS files_total,
+ S.param10 AS files_synced,
+ CASE S.param11 WHEN 0 THEN 'false'
+ WHEN 1 THEN 'true'
+ END AS new_requests
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
+
CREATE VIEW pg_user_mappings AS
SELECT
U.oid AS umid,
diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
index c937c39f50..79cae7e9ca 100644
--- a/src/backend/postmaster/checkpointer.c
+++ b/src/backend/postmaster/checkpointer.c
@@ -39,6 +39,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
#include "access/xlogrecovery.h"
+#include "commands/progress.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -163,7 +164,7 @@ static pg_time_t last_xlog_switch_time;
static void HandleCheckpointerInterrupts(void);
static void CheckArchiveTimeout(void);
static bool IsCheckpointOnSchedule(double progress);
-static bool ImmediateCheckpointRequested(void);
+static bool ImmediateCheckpointRequested(int flags);
static bool CompactCheckpointerRequestQueue(void);
static void UpdateSharedMemoryConfig(void);
@@ -667,16 +668,24 @@ CheckArchiveTimeout(void)
* there is one pending behind it.)
*/
static bool
-ImmediateCheckpointRequested(void)
+ImmediateCheckpointRequested(int flags)
{
volatile CheckpointerShmemStruct *cps = CheckpointerShmem;
+ if (cps->ckpt_flags & CHECKPOINT_REQUESTED)
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_NEW_REQUESTS, true);
+
/*
* We don't need to acquire the ckpt_lck in this case because we're only
* looking at a single flag bit.
*/
if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
+ {
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FLAGS,
+ (flags | CHECKPOINT_IMMEDIATE));
return true;
+ }
+
return false;
}
@@ -708,7 +717,7 @@ CheckpointWriteDelay(int flags, double progress)
*/
if (!(flags & CHECKPOINT_IMMEDIATE) &&
!ShutdownRequestPending &&
- !ImmediateCheckpointRequested() &&
+ !ImmediateCheckpointRequested(flags) &&
IsCheckpointOnSchedule(progress))
{
if (ConfigReloadPending)
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index ae13011d27..55f03c1301 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -39,6 +39,7 @@
#include "catalog/catalog.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2019,6 +2020,8 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_TOTAL,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2136,6 +2139,8 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2156,6 +2161,8 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.buf_written_checkpoints++;
num_written++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index e1fb631003..3acbf94c5e 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -368,6 +369,9 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_TOTAL,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -431,6 +435,8 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 893690dad5..c0de766fde 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -476,6 +476,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..33a64d2f0b 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,42 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_TYPE 0
+#define PROGRESS_CHECKPOINT_FLAGS 1
+#define PROGRESS_CHECKPOINT_LSN 2
+#define PROGRESS_CHECKPOINT_START_TIMESTAMP 3
+#define PROGRESS_CHECKPOINT_PHASE 4
+#define PROGRESS_CHECKPOINT_BUFFERS_TOTAL 5
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 6
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 7
+#define PROGRESS_CHECKPOINT_FILES_TOTAL 8
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 9
+#define PROGRESS_CHECKPOINT_NEW_REQUESTS 10
+
+/* Types of checkpoint (as advertised via PROGRESS_CHECKPOINT_TYPE) */
+#define PROGRESS_CHECKPOINT_TYPE_CHECKPOINT 1
+#define PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT 2
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 1
+#define PROGRESS_CHECKPOINT_PHASE_GET_VIRTUAL_TRANSACTION_IDS 2
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 3
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 4
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 5
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_ORIGIN 6
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 9
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 10
+#define PROGRESS_CHECKPOINT_PHASE_PREDICATE_LOCK_PAGES 11
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 12
+#define PROGRESS_CHECKPOINT_PHASE_SYNC_FILES 13
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 14
+#define PROGRESS_CHECKPOINT_PHASE_POST_CHECKPOINT_CLEANUP 15
+#define PROGRESS_CHECKPOINT_PHASE_INVALIDATE_REPLI_SLOTS 16
+#define PROGRESS_CHECKPOINT_PHASE_RECYCLE_OLD_XLOG 17
+#define PROGRESS_CHECKPOINT_PHASE_TRUNCATE_SUBTRANS 18
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 19
+
#endif
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index fc3cde3226..ec130dad2a 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1912,6 +1912,76 @@ pg_stat_progress_basebackup| SELECT s.pid,
s.param4 AS tablespaces_total,
s.param5 AS tablespaces_streamed
FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
+pg_stat_progress_checkpoint| SELECT s.pid,
+ CASE s.param1
+ WHEN 1 THEN 'checkpoint'::text
+ WHEN 2 THEN 'restartpoint'::text
+ ELSE NULL::text
+ END AS type,
+ (((((
+ CASE
+ WHEN ((s.param2 & (4)::bigint) > 0) THEN 'immediate '::text
+ ELSE ''::text
+ END ||
+ CASE
+ WHEN ((s.param2 & (8)::bigint) > 0) THEN 'force '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (16)::bigint) > 0) THEN 'flush-all '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (32)::bigint) > 0) THEN 'wait '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (128)::bigint) > 0) THEN 'wal '::text
+ ELSE ''::text
+ END) ||
+ CASE
+ WHEN ((s.param2 & (256)::bigint) > 0) THEN 'time '::text
+ ELSE ''::text
+ END) AS flags,
+ ('0/0'::pg_lsn + (
+ CASE
+ WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (s.param3)::numeric)) AS start_lsn,
+ to_timestamp(((946684800)::double precision + ((s.param4)::double precision / (1000000)::double precision))) AS start_time,
+ CASE s.param5
+ WHEN 1 THEN 'initializing'::text
+ WHEN 2 THEN 'getting virtual transaction IDs'::text
+ WHEN 3 THEN 'checkpointing replication slots'::text
+ WHEN 4 THEN 'checkpointing logical replication snapshot files'::text
+ WHEN 5 THEN 'checkpointing logical rewrite mapping files'::text
+ WHEN 6 THEN 'checkpointing replication origin'::text
+ WHEN 7 THEN 'checkpointing commit log pages'::text
+ WHEN 8 THEN 'checkpointing commit time stamp pages'::text
+ WHEN 9 THEN 'checkpointing subtransaction pages'::text
+ WHEN 10 THEN 'checkpointing multixact pages'::text
+ WHEN 11 THEN 'checkpointing predicate lock pages'::text
+ WHEN 12 THEN 'checkpointing buffers'::text
+ WHEN 13 THEN 'processing file sync requests'::text
+ WHEN 14 THEN 'performing two phase checkpoint'::text
+ WHEN 15 THEN 'performing post checkpoint cleanup'::text
+ WHEN 16 THEN 'invalidating replication slots'::text
+ WHEN 17 THEN 'recycling old WAL files'::text
+ WHEN 18 THEN 'truncating subtransactions'::text
+ WHEN 19 THEN 'finalizing'::text
+ ELSE NULL::text
+ END AS phase,
+ s.param6 AS buffers_total,
+ s.param7 AS buffers_processed,
+ s.param8 AS buffers_written,
+ s.param9 AS files_total,
+ s.param10 AS files_synced,
+ CASE s.param11
+ WHEN 0 THEN 'false'::text
+ WHEN 1 THEN 'true'::text
+ ELSE NULL::text
+ END AS new_requests
+ FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
pg_stat_progress_cluster| SELECT s.pid,
s.datid,
d.datname,
--
2.25.1
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-23 18:39 ` Alvaro Herrera <[email protected]>
3 siblings, 0 replies; 85+ messages in thread
From: Alvaro Herrera @ 2022-02-23 18:39 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
I think the change to ImmediateCheckpointRequested() makes no sense.
Before this patch, that function merely inquires whether there's an
immediate checkpoint queued. After this patch, it ... changes a
progress-reporting flag? I think it would make more sense to make the
progress-report flag change in whatever is the place that *requests* an
immediate checkpoint rather than here.
I think the use of capitals in CHECKPOINT and CHECKPOINTER in the
documentation is excessive. (Same for terms such as MULTIXACT and
others in those docs; we typically use those in lowercase when
user-facing; and do we really use term CLOG anymore? Don't we call it
"commit log" nowadays?)
--
Álvaro Herrera 39°49'30"S 73°17'W — https://www.EnterpriseDB.com/
"Hay quien adquiere la mala costumbre de ser infeliz" (M. A. Evans)
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-23 19:22 ` Justin Pryzby <[email protected]>
3 siblings, 0 replies; 85+ messages in thread
From: Justin Pryzby @ 2022-02-23 19:22 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; [email protected]
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
Maybe it should show a single row , unless the checkpointer isn't running at
all (like in single user mode).
+ Process ID of a CHECKPOINTER process.
It's *the* checkpointer process.
pgstatfuncs.c has a whitespace issue (tab-space).
I suppose the functions should set provolatile.
--
Justin
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-24 17:14 ` Matthias van de Meent <[email protected]>
2022-02-24 18:53 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
3 siblings, 1 reply; 85+ messages in thread
From: Matthias van de Meent @ 2022-02-24 17:14 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, 23 Feb 2022 at 14:28, Nitin Jadhav
<[email protected]> wrote:
>
> Sharing the v2 patch. Kindly have a look and share your comments.
Thanks for updating.
> diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
With the new pg_stat_progress_checkpoint, you should also add a
backreference to this progress reporting in the CHECKPOINT sql command
documentation located in checkpoint.sgml, and maybe in wal.sgml and/or
backup.sgml too. See e.g. cluster.sgml around line 195 for an example.
> diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
> +ImmediateCheckpointRequested(int flags)
> if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
> + {
> + updated_flags |= CHECKPOINT_IMMEDIATE;
I don't think that these changes are expected behaviour. Under in this
condition; the currently running checkpoint is still not 'immediate',
but it is going to hurry up for a new, actually immediate checkpoint.
Those are different kinds of checkpoint handling; and I don't think
you should modify the reported flags to show that we're going to do
stuff faster than usual. Maybe maintiain a seperate 'upcoming
checkpoint flags' field instead?
> diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
> + ( SELECT '0/0'::pg_lsn +
> + ((CASE
> + WHEN stat.lsn_int64 < 0 THEN pow(2::numeric, 64::numeric)::numeric
> + ELSE 0::numeric
> + END) +
> + stat.lsn_int64::numeric)
> + FROM (SELECT s.param3::bigint) AS stat(lsn_int64)
> + ) AS start_lsn,
My LSN select statement was an example that could be run directly in
psql; the so you didn't have to embed the SELECT into the view query.
The following should be sufficient (and save the planner a few cycles
otherwise spent in inlining):
+ ('0/0'::pg_lsn +
+ ((CASE
+ WHEN s.param3 < 0 THEN pow(2::numeric,
64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ s.param3::numeric)
+ ) AS start_lsn,
> diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
> +checkpoint_progress_start(int flags)
> [...]
> +checkpoint_progress_update_param(int index, int64 val)
> [...]
> +checkpoint_progress_end(void)
> +{
> + /* In bootstrap mode, we don't actually record anything. */
> + if (IsBootstrapProcessingMode())
> + return;
Disabling pgstat progress reporting when in bootstrap processing mode
/ startup/end-of-recovery makes very little sense (see upthread) and
should be removed, regardless of whether seperate functions stay.
> diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
> +#define PROGRESS_CHECKPOINT_PHASE_INIT 0
Generally, enum-like values in a stat_progress field are 1-indexed, to
differentiate between empty/uninitialized (0) and states that have
been set by the progress reporting infrastructure.
Kind regards,
Matthias van de Meent
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-24 17:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
@ 2022-02-24 18:53 ` Nitin Jadhav <[email protected]>
2022-02-25 07:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-25 14:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
0 siblings, 2 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-24 18:53 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> I think the change to ImmediateCheckpointRequested() makes no sense.
> Before this patch, that function merely inquires whether there's an
> immediate checkpoint queued. After this patch, it ... changes a
> progress-reporting flag? I think it would make more sense to make the
> progress-report flag change in whatever is the place that *requests* an
> immediate checkpoint rather than here.
>
> > diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
> > +ImmediateCheckpointRequested(int flags)
> > if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
> > + {
> > + updated_flags |= CHECKPOINT_IMMEDIATE;
>
> I don't think that these changes are expected behaviour. Under in this
> condition; the currently running checkpoint is still not 'immediate',
> but it is going to hurry up for a new, actually immediate checkpoint.
> Those are different kinds of checkpoint handling; and I don't think
> you should modify the reported flags to show that we're going to do
> stuff faster than usual. Maybe maintiain a seperate 'upcoming
> checkpoint flags' field instead?
Thank you Alvaro and Matthias for your views. I understand your point
of not updating the progress-report flag here as it just checks
whether the CHECKPOINT_IMMEDIATE is set or not and takes an action
based on that but it doesn't change the checkpoint flags. I will
modify the code but I am a bit confused here. As per Alvaro, we need
to make the progress-report flag change in whatever is the place that
*requests* an immediate checkpoint. I feel this gives information
about the upcoming checkpoint not the current one. So updating here
provides wrong details in the view. The flags available during
CreateCheckPoint() will remain same for the entire checkpoint
operation and we should show the same information in the view till it
completes. So just removing the above piece of code (modified in
ImmediateCheckpointRequested()) in the patch will make it correct. My
opinion about maintaining a separate field to show upcoming checkpoint
flags is it makes the view complex. Please share your thoughts.
Thanks & Regards,
On Thu, Feb 24, 2022 at 10:45 PM Matthias van de Meent
<[email protected]> wrote:
>
> On Wed, 23 Feb 2022 at 14:28, Nitin Jadhav
> <[email protected]> wrote:
> >
> > Sharing the v2 patch. Kindly have a look and share your comments.
>
> Thanks for updating.
>
> > diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
>
> With the new pg_stat_progress_checkpoint, you should also add a
> backreference to this progress reporting in the CHECKPOINT sql command
> documentation located in checkpoint.sgml, and maybe in wal.sgml and/or
> backup.sgml too. See e.g. cluster.sgml around line 195 for an example.
>
> > diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
> > +ImmediateCheckpointRequested(int flags)
> > if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
> > + {
> > + updated_flags |= CHECKPOINT_IMMEDIATE;
>
> I don't think that these changes are expected behaviour. Under in this
> condition; the currently running checkpoint is still not 'immediate',
> but it is going to hurry up for a new, actually immediate checkpoint.
> Those are different kinds of checkpoint handling; and I don't think
> you should modify the reported flags to show that we're going to do
> stuff faster than usual. Maybe maintiain a seperate 'upcoming
> checkpoint flags' field instead?
>
> > diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
> > + ( SELECT '0/0'::pg_lsn +
> > + ((CASE
> > + WHEN stat.lsn_int64 < 0 THEN pow(2::numeric, 64::numeric)::numeric
> > + ELSE 0::numeric
> > + END) +
> > + stat.lsn_int64::numeric)
> > + FROM (SELECT s.param3::bigint) AS stat(lsn_int64)
> > + ) AS start_lsn,
>
> My LSN select statement was an example that could be run directly in
> psql; the so you didn't have to embed the SELECT into the view query.
> The following should be sufficient (and save the planner a few cycles
> otherwise spent in inlining):
>
> + ('0/0'::pg_lsn +
> + ((CASE
> + WHEN s.param3 < 0 THEN pow(2::numeric,
> 64::numeric)::numeric
> + ELSE 0::numeric
> + END) +
> + s.param3::numeric)
> + ) AS start_lsn,
>
>
> > diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
> > +checkpoint_progress_start(int flags)
> > [...]
> > +checkpoint_progress_update_param(int index, int64 val)
> > [...]
> > +checkpoint_progress_end(void)
> > +{
> > + /* In bootstrap mode, we don't actually record anything. */
> > + if (IsBootstrapProcessingMode())
> > + return;
>
> Disabling pgstat progress reporting when in bootstrap processing mode
> / startup/end-of-recovery makes very little sense (see upthread) and
> should be removed, regardless of whether seperate functions stay.
>
> > diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
> > +#define PROGRESS_CHECKPOINT_PHASE_INIT 0
>
> Generally, enum-like values in a stat_progress field are 1-indexed, to
> differentiate between empty/uninitialized (0) and states that have
> been set by the progress reporting infrastructure.
>
>
>
> Kind regards,
>
> Matthias van de Meent
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-24 17:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-24 18:53 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-25 07:03 ` Julien Rouhaud <[email protected]>
2022-02-25 15:23 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
1 sibling, 1 reply; 85+ messages in thread
From: Julien Rouhaud @ 2022-02-25 07:03 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On Fri, Feb 25, 2022 at 12:23:27AM +0530, Nitin Jadhav wrote:
> > I think the change to ImmediateCheckpointRequested() makes no sense.
> > Before this patch, that function merely inquires whether there's an
> > immediate checkpoint queued. After this patch, it ... changes a
> > progress-reporting flag? I think it would make more sense to make the
> > progress-report flag change in whatever is the place that *requests* an
> > immediate checkpoint rather than here.
> >
> > > diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
> > > +ImmediateCheckpointRequested(int flags)
> > > if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
> > > + {
> > > + updated_flags |= CHECKPOINT_IMMEDIATE;
> >
> > I don't think that these changes are expected behaviour. Under in this
> > condition; the currently running checkpoint is still not 'immediate',
> > but it is going to hurry up for a new, actually immediate checkpoint.
> > Those are different kinds of checkpoint handling; and I don't think
> > you should modify the reported flags to show that we're going to do
> > stuff faster than usual. Maybe maintiain a seperate 'upcoming
> > checkpoint flags' field instead?
>
> Thank you Alvaro and Matthias for your views. I understand your point
> of not updating the progress-report flag here as it just checks
> whether the CHECKPOINT_IMMEDIATE is set or not and takes an action
> based on that but it doesn't change the checkpoint flags. I will
> modify the code but I am a bit confused here. As per Alvaro, we need
> to make the progress-report flag change in whatever is the place that
> *requests* an immediate checkpoint. I feel this gives information
> about the upcoming checkpoint not the current one. So updating here
> provides wrong details in the view. The flags available during
> CreateCheckPoint() will remain same for the entire checkpoint
> operation and we should show the same information in the view till it
> completes.
I'm not sure what Matthias meant, but as far as I know there's no fundamental
difference between checkpoint with and without the CHECKPOINT_IMMEDIATE flag,
and there's also no scheduling for multiple checkpoints.
Yes, the flags will remain the same but checkpoint.c will test both the passed
flags and the shmem flags to see whether a delay should be added or not, which
is the only difference in checkpoint processing for this flag. See the call to
ImmediateCheckpointRequested() which will look at the value in shmem:
/*
* Perform the usual duties and take a nap, unless we're behind schedule,
* in which case we just try to catch up as quickly as possible.
*/
if (!(flags & CHECKPOINT_IMMEDIATE) &&
!ShutdownRequestPending &&
!ImmediateCheckpointRequested() &&
IsCheckpointOnSchedule(progress))
[...]
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-24 17:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-24 18:53 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-25 07:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
@ 2022-02-25 15:23 ` Nitin Jadhav <[email protected]>
2022-02-25 16:35 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
0 siblings, 1 reply; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-25 15:23 UTC (permalink / raw)
To: Julien Rouhaud <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> > Thank you Alvaro and Matthias for your views. I understand your point
> > of not updating the progress-report flag here as it just checks
> > whether the CHECKPOINT_IMMEDIATE is set or not and takes an action
> > based on that but it doesn't change the checkpoint flags. I will
> > modify the code but I am a bit confused here. As per Alvaro, we need
> > to make the progress-report flag change in whatever is the place that
> > *requests* an immediate checkpoint. I feel this gives information
> > about the upcoming checkpoint not the current one. So updating here
> > provides wrong details in the view. The flags available during
> > CreateCheckPoint() will remain same for the entire checkpoint
> > operation and we should show the same information in the view till it
> > completes.
>
> I'm not sure what Matthias meant, but as far as I know there's no fundamental
> difference between checkpoint with and without the CHECKPOINT_IMMEDIATE flag,
> and there's also no scheduling for multiple checkpoints.
>
> Yes, the flags will remain the same but checkpoint.c will test both the passed
> flags and the shmem flags to see whether a delay should be added or not, which
> is the only difference in checkpoint processing for this flag. See the call to
> ImmediateCheckpointRequested() which will look at the value in shmem:
>
> /*
> * Perform the usual duties and take a nap, unless we're behind schedule,
> * in which case we just try to catch up as quickly as possible.
> */
> if (!(flags & CHECKPOINT_IMMEDIATE) &&
> !ShutdownRequestPending &&
> !ImmediateCheckpointRequested() &&
> IsCheckpointOnSchedule(progress))
I understand that the checkpointer considers flags as well as the
shmem flags and if CHECKPOINT_IMMEDIATE flag is set, it affects the
current checkpoint operation (No further delay) but does not change
the current flag value. Should we display this change in the kind
field of the view or not? Please share your thoughts.
Thanks & Regards,
Nitin Jadhav
On Fri, Feb 25, 2022 at 12:33 PM Julien Rouhaud <[email protected]> wrote:
>
> Hi,
>
> On Fri, Feb 25, 2022 at 12:23:27AM +0530, Nitin Jadhav wrote:
> > > I think the change to ImmediateCheckpointRequested() makes no sense.
> > > Before this patch, that function merely inquires whether there's an
> > > immediate checkpoint queued. After this patch, it ... changes a
> > > progress-reporting flag? I think it would make more sense to make the
> > > progress-report flag change in whatever is the place that *requests* an
> > > immediate checkpoint rather than here.
> > >
> > > > diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
> > > > +ImmediateCheckpointRequested(int flags)
> > > > if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
> > > > + {
> > > > + updated_flags |= CHECKPOINT_IMMEDIATE;
> > >
> > > I don't think that these changes are expected behaviour. Under in this
> > > condition; the currently running checkpoint is still not 'immediate',
> > > but it is going to hurry up for a new, actually immediate checkpoint.
> > > Those are different kinds of checkpoint handling; and I don't think
> > > you should modify the reported flags to show that we're going to do
> > > stuff faster than usual. Maybe maintiain a seperate 'upcoming
> > > checkpoint flags' field instead?
> >
> > Thank you Alvaro and Matthias for your views. I understand your point
> > of not updating the progress-report flag here as it just checks
> > whether the CHECKPOINT_IMMEDIATE is set or not and takes an action
> > based on that but it doesn't change the checkpoint flags. I will
> > modify the code but I am a bit confused here. As per Alvaro, we need
> > to make the progress-report flag change in whatever is the place that
> > *requests* an immediate checkpoint. I feel this gives information
> > about the upcoming checkpoint not the current one. So updating here
> > provides wrong details in the view. The flags available during
> > CreateCheckPoint() will remain same for the entire checkpoint
> > operation and we should show the same information in the view till it
> > completes.
>
> I'm not sure what Matthias meant, but as far as I know there's no fundamental
> difference between checkpoint with and without the CHECKPOINT_IMMEDIATE flag,
> and there's also no scheduling for multiple checkpoints.
>
> Yes, the flags will remain the same but checkpoint.c will test both the passed
> flags and the shmem flags to see whether a delay should be added or not, which
> is the only difference in checkpoint processing for this flag. See the call to
> ImmediateCheckpointRequested() which will look at the value in shmem:
>
> /*
> * Perform the usual duties and take a nap, unless we're behind schedule,
> * in which case we just try to catch up as quickly as possible.
> */
> if (!(flags & CHECKPOINT_IMMEDIATE) &&
> !ShutdownRequestPending &&
> !ImmediateCheckpointRequested() &&
> IsCheckpointOnSchedule(progress))
> [...]
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-24 17:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-24 18:53 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-25 07:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-25 15:23 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-25 16:35 ` Julien Rouhaud <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Julien Rouhaud @ 2022-02-25 16:35 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Matthias van de Meent <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
On Fri, Feb 25, 2022 at 08:53:50PM +0530, Nitin Jadhav wrote:
> >
> > I'm not sure what Matthias meant, but as far as I know there's no fundamental
> > difference between checkpoint with and without the CHECKPOINT_IMMEDIATE flag,
> > and there's also no scheduling for multiple checkpoints.
> >
> > Yes, the flags will remain the same but checkpoint.c will test both the passed
> > flags and the shmem flags to see whether a delay should be added or not, which
> > is the only difference in checkpoint processing for this flag. See the call to
> > ImmediateCheckpointRequested() which will look at the value in shmem:
> >
> > /*
> > * Perform the usual duties and take a nap, unless we're behind schedule,
> > * in which case we just try to catch up as quickly as possible.
> > */
> > if (!(flags & CHECKPOINT_IMMEDIATE) &&
> > !ShutdownRequestPending &&
> > !ImmediateCheckpointRequested() &&
> > IsCheckpointOnSchedule(progress))
>
> I understand that the checkpointer considers flags as well as the
> shmem flags and if CHECKPOINT_IMMEDIATE flag is set, it affects the
> current checkpoint operation (No further delay) but does not change
> the current flag value. Should we display this change in the kind
> field of the view or not? Please share your thoughts.
I think the fields should be added. It's good to know that a checkpoint was
trigger due to normal activity and should be spreaded, and then something
upgraded it to an immediate checkpoint. If you're desperately waiting for the
end of a checkpoint for some reason and ask for an immediate checkpoint, you'll
certainly be happy to see that the checkpointer is aware of it.
But maybe I missed something in the code, so let's wait for Matthias input
about it.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-24 17:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-24 18:53 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-25 14:56 ` Nitin Jadhav <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-25 14:56 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> Thank you Alvaro and Matthias for your views. I understand your point
> of not updating the progress-report flag here as it just checks
> whether the CHECKPOINT_IMMEDIATE is set or not and takes an action
> based on that but it doesn't change the checkpoint flags. I will
> modify the code but I am a bit confused here. As per Alvaro, we need
> to make the progress-report flag change in whatever is the place that
> *requests* an immediate checkpoint. I feel this gives information
> about the upcoming checkpoint not the current one. So updating here
> provides wrong details in the view. The flags available during
> CreateCheckPoint() will remain same for the entire checkpoint
> operation and we should show the same information in the view till it
> completes. So just removing the above piece of code (modified in
> ImmediateCheckpointRequested()) in the patch will make it correct. My
> opinion about maintaining a separate field to show upcoming checkpoint
> flags is it makes the view complex. Please share your thoughts.
I have modified the code accordingly.
---
> I think the use of capitals in CHECKPOINT and CHECKPOINTER in the
> documentation is excessive.
Fixed. Here the word CHECKPOINT represents command/checkpoint
operation. If we treat it as a checkpoint operation, I agree to use
lowercase but if we treat it as command, then I think uppercase is
recommended (Refer
https://www.postgresql.org/docs/14/sql-checkpoint.html). Is it ok to
always use lowercase here?
---
> (Same for terms such as MULTIXACT and
> others in those docs; we typically use those in lowercase when
> user-facing; and do we really use term CLOG anymore? Don't we call it
> "commit log" nowadays?)
I have observed the CLOG term in the existing documentation. Anyways I
have changed MULTIXACT to multixact, SUBTRANS to subtransaction and
CLOG to commit log.
---
> + Whenever the checkpoint operation is running, the
> + <structname>pg_stat_progress_checkpoint</structname> view will contain a
> + single row indicating the progress of the checkpoint. The tables below
>
> Maybe it should show a single row , unless the checkpointer isn't running at
> all (like in single user mode).
Nice thought. Can we add an additional checkpoint phase like 'Idle'.
Idle is ON whenever the checkpointer process is running and there are
no on-going checkpoint Thoughts?
---
> + Process ID of a CHECKPOINTER process.
>
> It's *the* checkpointer process.
Fixed.
---
> pgstatfuncs.c has a whitespace issue (tab-space).
I have verified with 'git diff --check' and also manually. I did not
find any issue. Kindly mention the specific code which has an issue.
---
> I suppose the functions should set provolatile.
Fixed.
---
> > I am storing the checkpoint start timestamp in the st_progress_param[]
> > and this gets set only once during the checkpoint (at the start of the
> > checkpoint). I have added function
> > pg_stat_get_progress_checkpoint_elapsed() which calculates the elapsed
> > time and returns a string. This function gets called whenever
> > pg_stat_progress_checkpoint view is queried. Kindly refer v2 patch and
> > share your thoughts.
>
> I dislike the lack of access to the actual value of the checkpoint
> start / checkpoint elapsed field.
>
> As a user, if I query the pg_stat_progress_* views, my terminal or
> application can easily interpret an `interval` value and cast it to
> string, but the opposite is not true: the current implementation for
> pg_stat_get_progress_checkpoint_elapsed loses precision. This is why
> we use typed numeric fields in effectively all other places instead of
> stringified versions of the values: oid fields, counters, etc are all
> rendered as bigint in the view, so that no information is lost and
> interpretation is trivial.
Displaying start time of the checkpoint.
---
> > I understand that the log based reporting is very costly and very
> > frequent updates are not advisable. I am planning to use the existing
> > infrastructure of 'log_startup_progress_interval' which provides an
> > option for the user to configure the interval between each progress
> > update. Hence it avoids frequent updates to server logs. This approach
> > is used only during shutdown and end-of-recovery cases because we
> > cannot access pg_stat_progress_checkpoint view during those scenarios.
>
> I see; but log_startup_progress_interval seems to be exclusively
> consumed through the ereport_startup_progress macro. Why put
> startup/shutdown logging on the same path as the happy flow of normal
> checkpoints?
You mean to say while updating the progress of the checkpoint, call
pgstat_progress_update_param() and then call
ereport_startup_progress() ?
> I think that, instead of looking to what might at some point be added,
> it is better to use the currently available functions instead, and
> move to new functions if and when the log-based reporting requires it.
Make sense. Removing checkpoint_progress_update_param() and
checkpoint_progress_end(). I would like to concentrate on
pg_stat_progress_checkpoint view as of now and I will consider log
based reporting later.
> > diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
>
> With the new pg_stat_progress_checkpoint, you should also add a
> backreference to this progress reporting in the CHECKPOINT sql command
> documentation located in checkpoint.sgml, and maybe in wal.sgml and/or
> backup.sgml too. See e.g. cluster.sgml around line 195 for an example.
I have updated in checkpoint.sqml and wal.sqml.
> > diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
> > + ( SELECT '0/0'::pg_lsn +
> > + ((CASE
> > + WHEN stat.lsn_int64 < 0 THEN pow(2::numeric, 64::numeric)::numeric
> > + ELSE 0::numeric
> > + END) +
> > + stat.lsn_int64::numeric)
> > + FROM (SELECT s.param3::bigint) AS stat(lsn_int64)
> > + ) AS start_lsn,
>
> My LSN select statement was an example that could be run directly in
> psql; the so you didn't have to embed the SELECT into the view query.
> The following should be sufficient (and save the planner a few cycles
> otherwise spent in inlining):
>
> + ('0/0'::pg_lsn +
> + ((CASE
> + WHEN s.param3 < 0 THEN pow(2::numeric,
> 64::numeric)::numeric
> + ELSE 0::numeric
> + END) +
> + s.param3::numeric)
> + ) AS start_lsn,
Thanks for the suggestion. Fixed.
> > diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
> > +checkpoint_progress_start(int flags)
> > [...]
> > +checkpoint_progress_update_param(int index, int64 val)
> > [...]
> > +checkpoint_progress_end(void)
> > +{
> > + /* In bootstrap mode, we don't actually record anything. */
> > + if (IsBootstrapProcessingMode())
> > + return;
>
> Disabling pgstat progress reporting when in bootstrap processing mode
> / startup/end-of-recovery makes very little sense (see upthread) and
> should be removed, regardless of whether seperate functions stay.
Removed since log based reporting is not part of the current patch.
> > diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
> > +#define PROGRESS_CHECKPOINT_PHASE_INIT 0
>
> Generally, enum-like values in a stat_progress field are 1-indexed, to
> differentiate between empty/uninitialized (0) and states that have
> been set by the progress reporting infrastructure.
Fixed.
Please find the v3 patch attached and share your thoughts.
Thanks & Regards,
Nitin Jadhav
On Fri, Feb 25, 2022 at 12:23 AM Nitin Jadhav
<[email protected]> wrote:
>
> > I think the change to ImmediateCheckpointRequested() makes no sense.
> > Before this patch, that function merely inquires whether there's an
> > immediate checkpoint queued. After this patch, it ... changes a
> > progress-reporting flag? I think it would make more sense to make the
> > progress-report flag change in whatever is the place that *requests* an
> > immediate checkpoint rather than here.
> >
> > > diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
> > > +ImmediateCheckpointRequested(int flags)
> > > if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
> > > + {
> > > + updated_flags |= CHECKPOINT_IMMEDIATE;
> >
> > I don't think that these changes are expected behaviour. Under in this
> > condition; the currently running checkpoint is still not 'immediate',
> > but it is going to hurry up for a new, actually immediate checkpoint.
> > Those are different kinds of checkpoint handling; and I don't think
> > you should modify the reported flags to show that we're going to do
> > stuff faster than usual. Maybe maintiain a seperate 'upcoming
> > checkpoint flags' field instead?
>
> Thank you Alvaro and Matthias for your views. I understand your point
> of not updating the progress-report flag here as it just checks
> whether the CHECKPOINT_IMMEDIATE is set or not and takes an action
> based on that but it doesn't change the checkpoint flags. I will
> modify the code but I am a bit confused here. As per Alvaro, we need
> to make the progress-report flag change in whatever is the place that
> *requests* an immediate checkpoint. I feel this gives information
> about the upcoming checkpoint not the current one. So updating here
> provides wrong details in the view. The flags available during
> CreateCheckPoint() will remain same for the entire checkpoint
> operation and we should show the same information in the view till it
> completes. So just removing the above piece of code (modified in
> ImmediateCheckpointRequested()) in the patch will make it correct. My
> opinion about maintaining a separate field to show upcoming checkpoint
> flags is it makes the view complex. Please share your thoughts.
>
> Thanks & Regards,
>
> On Thu, Feb 24, 2022 at 10:45 PM Matthias van de Meent
> <[email protected]> wrote:
> >
> > On Wed, 23 Feb 2022 at 14:28, Nitin Jadhav
> > <[email protected]> wrote:
> > >
> > > Sharing the v2 patch. Kindly have a look and share your comments.
> >
> > Thanks for updating.
> >
> > > diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
> >
> > With the new pg_stat_progress_checkpoint, you should also add a
> > backreference to this progress reporting in the CHECKPOINT sql command
> > documentation located in checkpoint.sgml, and maybe in wal.sgml and/or
> > backup.sgml too. See e.g. cluster.sgml around line 195 for an example.
> >
> > > diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c
> > > +ImmediateCheckpointRequested(int flags)
> > > if (cps->ckpt_flags & CHECKPOINT_IMMEDIATE)
> > > + {
> > > + updated_flags |= CHECKPOINT_IMMEDIATE;
> >
> > I don't think that these changes are expected behaviour. Under in this
> > condition; the currently running checkpoint is still not 'immediate',
> > but it is going to hurry up for a new, actually immediate checkpoint.
> > Those are different kinds of checkpoint handling; and I don't think
> > you should modify the reported flags to show that we're going to do
> > stuff faster than usual. Maybe maintiain a seperate 'upcoming
> > checkpoint flags' field instead?
> >
> > > diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
> > > + ( SELECT '0/0'::pg_lsn +
> > > + ((CASE
> > > + WHEN stat.lsn_int64 < 0 THEN pow(2::numeric, 64::numeric)::numeric
> > > + ELSE 0::numeric
> > > + END) +
> > > + stat.lsn_int64::numeric)
> > > + FROM (SELECT s.param3::bigint) AS stat(lsn_int64)
> > > + ) AS start_lsn,
> >
> > My LSN select statement was an example that could be run directly in
> > psql; the so you didn't have to embed the SELECT into the view query.
> > The following should be sufficient (and save the planner a few cycles
> > otherwise spent in inlining):
> >
> > + ('0/0'::pg_lsn +
> > + ((CASE
> > + WHEN s.param3 < 0 THEN pow(2::numeric,
> > 64::numeric)::numeric
> > + ELSE 0::numeric
> > + END) +
> > + s.param3::numeric)
> > + ) AS start_lsn,
> >
> >
> > > diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
> > > +checkpoint_progress_start(int flags)
> > > [...]
> > > +checkpoint_progress_update_param(int index, int64 val)
> > > [...]
> > > +checkpoint_progress_end(void)
> > > +{
> > > + /* In bootstrap mode, we don't actually record anything. */
> > > + if (IsBootstrapProcessingMode())
> > > + return;
> >
> > Disabling pgstat progress reporting when in bootstrap processing mode
> > / startup/end-of-recovery makes very little sense (see upthread) and
> > should be removed, regardless of whether seperate functions stay.
> >
> > > diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
> > > +#define PROGRESS_CHECKPOINT_PHASE_INIT 0
> >
> > Generally, enum-like values in a stat_progress field are 1-indexed, to
> > differentiate between empty/uninitialized (0) and states that have
> > been set by the progress reporting infrastructure.
> >
> >
> >
> > Kind regards,
> >
> > Matthias van de Meent
Attachments:
[application/octet-stream] v3-0001-pg_stat_progress_checkpoint-view.patch (32.2K, ../../CAMm1aWbSC091NH1wRKZo=tO-F2WoNuYptpmP1bXEyyX+7YQvvw@mail.gmail.com/2-v3-0001-pg_stat_progress_checkpoint-view.patch)
download | inline diff:
From d2e0358e4d3c498afcc7ddaa7a84a7d79e921008 Mon Sep 17 00:00:00 2001
From: Nitin Jadhav <[email protected]>
Date: Fri, 25 Feb 2022 14:47:59 +0000
Subject: [PATCH] pg_stat_progress_checkpoint view
---
doc/src/sgml/monitoring.sgml | 357 +++++++++++++++++++++++++++
doc/src/sgml/ref/checkpoint.sgml | 6 +
doc/src/sgml/wal.sgml | 5 +-
src/backend/access/transam/xlog.c | 79 ++++++
src/backend/catalog/system_views.sql | 35 +++
src/backend/storage/buffer/bufmgr.c | 7 +
src/backend/storage/sync/sync.c | 6 +
src/backend/utils/adt/pgstatfuncs.c | 49 ++++
src/include/catalog/pg_proc.dat | 12 +
src/include/commands/progress.h | 28 +++
src/include/utils/backend_progress.h | 3 +-
src/test/regress/expected/rules.out | 32 +++
12 files changed, 617 insertions(+), 2 deletions(-)
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index bf7625d988..6dfc9bc392 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -401,6 +401,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
See <xref linkend='copy-progress-reporting'/>.
</entry>
</row>
+
+ <row>
+ <entry><structname>pg_stat_progress_checkpoint</structname><indexterm><primary>pg_stat_progress_checkpoint</primary></indexterm></entry>
+ <entry>One row only, showing the progress of the checkpoint.
+ See <xref linkend='checkpoint-progress-reporting'/>.
+ </entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -6895,6 +6902,356 @@ SELECT pg_stat_get_backend_pid(s.backendid) AS pid,
</table>
</sect2>
+ <sect2 id="checkpoint-progress-reporting">
+ <title>Checkpoint Progress Reporting</title>
+
+ <indexterm>
+ <primary>pg_stat_progress_checkpoint</primary>
+ </indexterm>
+
+ <para>
+ Whenever the checkpoint operation is running, the
+ <structname>pg_stat_progress_checkpoint</structname> view will contain a
+ single row indicating the progress of the checkpoint. The tables below
+ describe the information that will be reported and provide information about
+ how to interpret it.
+ </para>
+
+ <table id="pg-stat-progress-checkpoint-view" xreflabel="pg_stat_progress_checkpoint">
+ <title><structname>pg_stat_progress_checkpoint</structname> View</title>
+ <tgroup cols="1">
+ <thead>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ Column Type
+ </para>
+ <para>
+ Description
+ </para></entry>
+ </row>
+ </thead>
+
+ <tbody>
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>pid</structfield> <type>integer</type>
+ </para>
+ <para>
+ Process ID of the checkpointer process.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>type</structfield> <type>text</type>
+ </para>
+ <para>
+ Type of checkpoint. See <xref linkend="checkpoint-types"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>kind</structfield> <type>text</type>
+ </para>
+ <para>
+ Kind of checkpoint. See <xref linkend="checkpoint-kinds"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_lsn</structfield> <type>text</type>
+ </para>
+ <para>
+ The checkpoint start location.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>start_time</structfield> <type>timestamp with time zone</type>
+ </para>
+ <para>
+ Elapsed time of the checkpoint.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>phase</structfield> <type>text</type>
+ </para>
+ <para>
+ Current processing phase. See <xref linkend="checkpoint-phases"/>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of buffers to be written. This is estimated and reported
+ as of the beginning of buffer write operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_processed</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers processed. This counter increases when the targeted
+ buffer is processed. This number will eventually become equal to
+ <literal> total_buffer_writes </literal> when the checkpoint is
+ complete.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>buffers_written</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of buffers written. This counter only advances when the targeted
+ buffers is written. Note that some of the buffers are processed but may
+ not required to be written. So this count will always be less than or
+ equal to <literal>total_buffer_writes</literal>.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_total</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Total number of files to be synced. This is estimated and reported as of
+ the beginning of sync operation.
+ </para></entry>
+ </row>
+
+ <row>
+ <entry role="catalog_table_entry"><para role="column_definition">
+ <structfield>files_synced</structfield> <type>bigint</type>
+ </para>
+ <para>
+ Number of files synced. This counter advances when the targeted file is
+ synced. This number will eventually become equal to
+ <literal>total_file_syncs</literal> when the checkpoint is complete.
+ </para></entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-types">
+ <title>Checkpoint types</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Types</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>checkpoint</literal></entry>
+ <entry>
+ The current operation is checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>restartpoint</literal></entry>
+ <entry>
+ The current operation is restartpoint.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-kinds">
+ <title>Checkpoint kinds</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Kinds</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>shutdown</literal></entry>
+ <entry>
+ The checkpoint is for shutdown.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>end-of-recovery</literal></entry>
+ <entry>
+ The checkpoint is for end-of-recovery.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>immediate</literal></entry>
+ <entry>
+ The checkpoint is happens without delays.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>force</literal></entry>
+ <entry>
+ The checkpoint is started because some operation forced a checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>flush all</literal></entry>
+ <entry>
+ The checkpoint flushes all pages, including those belonging to unlogged
+ tables.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wait</literal></entry>
+ <entry>
+ Wait for completion before returning.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>requested</literal></entry>
+ <entry>
+ The checkpoint request has been made.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>wal</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>max_wal_size</literal> is
+ reached.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>time</literal></entry>
+ <entry>
+ The checkpoint is started because <literal>checkpoint_timeout</literal>
+ expired.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ <table id="checkpoint-phases">
+ <title>Checkpoint phases</title>
+ <tgroup cols="2">
+ <colspec colname="col1" colwidth="1*"/>
+ <colspec colname="col2" colwidth="2*"/>
+ <thead>
+ <row>
+ <entry>Phase</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry><literal>initializing</literal></entry>
+ <entry>
+ The checkpointer process is preparing to begin the checkpoint operation.
+ This phase is expected to be very brief.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing replication slots</literal></entry>
+ <entry>
+ The checkpointer process is currently flushing all the replication slots
+ to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing snapshots</literal></entry>
+ <entry>
+ The checkpointer process is currently removing all the serialized
+ snapshots that are not required anymore.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing logical rewrite mappings</literal></entry>
+ <entry>
+ The checkpointer process is currently removing/flushing the logical
+ rewrite mappings.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit log pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit log pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing commit time stamp pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing commit time stamp pages to
+ disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing subtransaction pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing subtransaction pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing multixact pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing multixact pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing SLRU pages</literal></entry>
+ <entry>
+ The checkpointer process is currently writing SLRU pages to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>checkpointing buffers</literal></entry>
+ <entry>
+ The checkpointer process is currently writing buffers to disk.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing sync requests</literal></entry>
+ <entry>
+ The checkpointer process is currently performing sync requests.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>performing two phase checkpoint</literal></entry>
+ <entry>
+ The checkpointer process is currently performing two phase checkpoint.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>recycling old XLOG files</literal></entry>
+ <entry>
+ The checkpointer process is currently recycling old XLOG files.
+ </entry>
+ </row>
+ <row>
+ <entry><literal>finalizing</literal></entry>
+ <entry>
+ The checkpointer process is finalizing the checkpoint operation.
+ </entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+
+ </sect2>
+
</sect1>
<sect1 id="dynamic-trace">
diff --git a/doc/src/sgml/ref/checkpoint.sgml b/doc/src/sgml/ref/checkpoint.sgml
index 1cebc03d15..a88c76533a 100644
--- a/doc/src/sgml/ref/checkpoint.sgml
+++ b/doc/src/sgml/ref/checkpoint.sgml
@@ -56,6 +56,12 @@ CHECKPOINT
the <link linkend="predefined-roles-table"><literal>pg_checkpointer</literal></link>
role can call <command>CHECKPOINT</command>.
</para>
+
+ <para>
+ The checkpointer process running the checkpoint will report its progress
+ in the <structname>pg_stat_progress_checkpoint</structname> view. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
+ </para>
</refsect1>
<refsect1>
diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml
index 2bb27a8468..a75d1d63d0 100644
--- a/doc/src/sgml/wal.sgml
+++ b/doc/src/sgml/wal.sgml
@@ -530,7 +530,10 @@
adjust the <xref linkend="guc-archive-timeout"/> parameter rather than the
checkpoint parameters.)
It is also possible to force a checkpoint by using the SQL
- command <command>CHECKPOINT</command>.
+ command <command>CHECKPOINT</command>. The checkpointer process running the
+ checkpoint will report its progress in the
+ <structname>pg_stat_progress_checkpoint</structname> view. See
+ <xref linkend="checkpoint-progress-reporting"/> for details.
</para>
<para>
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 0d2bd7a357..af6e64c836 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -65,6 +65,7 @@
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
#include "catalog/pg_database.h"
+#include "commands/progress.h"
#include "common/controldata_utils.h"
#include "executor/instrument.h"
#include "miscadmin.h"
@@ -719,6 +720,8 @@ static void WALInsertLockAcquireExclusive(void);
static void WALInsertLockRelease(void);
static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt);
+static void checkpoint_progress_start(int flags);
+
/*
* Insert an XLOG record represented by an already-constructed chain of data
* chunks. This is a low-level routine; to construct the WAL record header
@@ -6296,6 +6299,9 @@ CreateCheckPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the checkpoint. */
+ checkpoint_progress_start(flags);
+
/*
* Use a critical section to force system panic if we have trouble.
*/
@@ -6394,6 +6400,7 @@ CreateCheckPoint(int flags)
curInsert += SizeOfXLogShortPHD;
}
checkPoint.redo = curInsert;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_LSN, checkPoint.redo);
/*
* Here we update the shared RedoRecPtr for future XLogInsert calls; this
@@ -6629,8 +6636,12 @@ CreateCheckPoint(int flags)
KeepLogSeg(recptr, &_logSegNo);
}
_logSegNo--;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, recptr,
checkPoint.ThisTimeLineID);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/*
* Make more log segments if needed. (Do this after recycling old log
@@ -6652,6 +6663,9 @@ CreateCheckPoint(int flags)
/* Real work is done; log and update stats. */
LogCheckpointEnd(false);
+ /* Stop reporting progress of the checkpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, false, true);
@@ -6808,29 +6822,60 @@ static void
CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
{
CheckPointRelationMap();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS);
CheckPointReplicationSlots();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS);
CheckPointSnapBuild();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS);
CheckPointLogicalRewriteHeap();
CheckPointReplicationOrigin();
/* Write out all dirty data in SLRUs and the main buffer pool */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES);
CheckPointCLOG();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES);
CheckPointCommitTs();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES);
CheckPointSUBTRANS();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES);
CheckPointMultiXact();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES);
CheckPointPredicate();
+
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_BUFFERS);
CheckPointBuffers(flags);
/* Perform all queued up fsyncs */
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FILE_SYNC);
ProcessSyncRequests();
CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
/* We deliberately delay 2PC checkpointing as long as possible */
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_TWO_PHASE);
CheckPointTwoPhase(checkPointRedo);
}
@@ -6977,6 +7022,9 @@ CreateRestartPoint(int flags)
MemSet(&CheckpointStats, 0, sizeof(CheckpointStats));
CheckpointStats.ckpt_start_t = GetCurrentTimestamp();
+ /* Prepare to report progress of the restartpoint. */
+ checkpoint_progress_start(flags);
+
if (log_checkpoints)
LogCheckpointStart(flags, true);
@@ -7077,7 +7125,11 @@ CreateRestartPoint(int flags)
if (!RecoveryInProgress())
replayTLI = XLogCtl->InsertTimeLineID;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE);
RemoveOldXlogFiles(_logSegNo, RedoRecPtr, endptr, replayTLI);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_PHASE_FINALIZE);
/*
* Make more log segments if needed. (Do this after recycling old log
@@ -7098,6 +7150,9 @@ CreateRestartPoint(int flags)
/* Real work is done; log and update stats. */
LogCheckpointEnd(true);
+ /* Stop reporting progress of the restartpoint. */
+ pgstat_progress_end_command();
+
/* Reset the process title */
update_checkpoint_display(flags, true, true);
@@ -9197,3 +9252,27 @@ SetWalWriterSleeping(bool sleeping)
XLogCtl->WalWriterSleeping = sleeping;
SpinLockRelease(&XLogCtl->info_lck);
}
+
+/*
+ * Start reporting progress of the checkpoint.
+ */
+static void
+checkpoint_progress_start(int flags)
+{
+ const int index[] = {
+ PROGRESS_CHECKPOINT_TIMELINE,
+ PROGRESS_CHECKPOINT_KIND,
+ PROGRESS_CHECKPOINT_PHASE,
+ PROGRESS_CHECKPOINT_START_TIMESTAMP
+ };
+ int64 val[4];
+
+ pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
+
+ val[0] = XLogCtl->InsertTimeLineID;
+ val[1] = flags;
+ val[2] = PROGRESS_CHECKPOINT_PHASE_INIT;
+ val[3] = CheckpointStats.ckpt_start_t;
+
+ pgstat_progress_update_multi_param(4, index, val);
+}
\ No newline at end of file
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 3cb69b1f87..1c4c8ead9c 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1286,3 +1286,38 @@ CREATE VIEW pg_stat_subscription_workers AS
FROM pg_subscription_rel) sr,
LATERAL pg_stat_get_subscription_worker(sr.subid, sr.relid) w
JOIN pg_subscription s ON (w.subid = s.oid);
+
+CREATE VIEW pg_stat_progress_checkpoint AS
+ SELECT
+ S.pid AS pid,
+ pg_stat_get_progress_checkpoint_type(S.param1) AS type,
+ pg_stat_get_progress_checkpoint_kind(S.param2) AS kind,
+ ( '0/0'::pg_lsn +
+ ((CASE
+ WHEN S.param3 < 0 THEN pow(2::numeric, 64::numeric)::numeric
+ ELSE 0::numeric
+ END) +
+ S.param3::numeric)
+ ) AS start_lsn,
+ pg_stat_get_progress_checkpoint_start_time(S.param4) AS start_time,
+ CASE S.param5 WHEN 1 THEN 'initializing'
+ WHEN 2 THEN 'checkpointing replication slots'
+ WHEN 3 THEN 'checkpointing snapshots'
+ WHEN 4 THEN 'checkpointing logical rewrite mappings'
+ WHEN 5 THEN 'checkpointing commit log pages'
+ WHEN 6 THEN 'checkpointing commit time stamp pages'
+ WHEN 7 THEN 'checkpointing subtransaction pages'
+ WHEN 8 THEN 'checkpointing multixact pages'
+ WHEN 9 THEN 'checkpointing SLRU pages'
+ WHEN 10 THEN 'checkpointing buffers'
+ WHEN 11 THEN 'performing sync requests'
+ WHEN 12 THEN 'performing two phase checkpoint'
+ WHEN 13 THEN 'recycling old XLOG files'
+ WHEN 14 THEN 'Finalizing'
+ END AS phase,
+ S.param6 AS total_buffer_writes,
+ S.param7 AS buffers_processed,
+ S.param8 AS buffers_written,
+ S.param9 AS total_file_syncs,
+ S.param10 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT') AS S;
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index f5459c68f8..9663035d7a 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -38,6 +38,7 @@
#include "access/xlogutils.h"
#include "catalog/catalog.h"
#include "catalog/storage.h"
+#include "commands/progress.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
@@ -2012,6 +2013,8 @@ BufferSync(int flags)
WritebackContextInit(&wb_context, &checkpoint_flush_after);
TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_TOTAL,
+ num_to_scan);
/*
* Sort buffers that need to be written to reduce the likelihood of random
@@ -2129,6 +2132,8 @@ BufferSync(int flags)
bufHdr = GetBufferDescriptor(buf_id);
num_processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_PROCESSED,
+ num_processed);
/*
* We don't need to acquire the lock here, because we're only looking
@@ -2149,6 +2154,8 @@ BufferSync(int flags)
TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
PendingCheckpointerStats.m_buf_written_checkpoints++;
num_written++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_BUFFERS_WRITTEN,
+ num_written);
}
}
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
index e161d57761..638d3eb781 100644
--- a/src/backend/storage/sync/sync.c
+++ b/src/backend/storage/sync/sync.c
@@ -23,6 +23,7 @@
#include "access/multixact.h"
#include "access/xlog.h"
#include "access/xlogutils.h"
+#include "commands/progress.h"
#include "commands/tablespace.h"
#include "miscadmin.h"
#include "pgstat.h"
@@ -356,6 +357,9 @@ ProcessSyncRequests(void)
/* Now scan the hashtable for fsync requests to process */
absorb_counter = FSYNCS_PER_ABSORB;
hash_seq_init(&hstat, pendingOps);
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_TOTAL,
+ hash_get_num_entries(pendingOps));
+
while ((entry = (PendingFsyncEntry *) hash_seq_search(&hstat)) != NULL)
{
int failures;
@@ -419,6 +423,8 @@ ProcessSyncRequests(void)
longest = elapsed;
total_elapsed += elapsed;
processed++;
+ pgstat_progress_update_param(PROGRESS_CHECKPOINT_FILES_SYNCED,
+ processed);
if (log_checkpoints)
elog(DEBUG1, "checkpoint sync: number=%d file=%s time=%.3f ms",
diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c
index 30e8dfa7c1..2ea390a2b6 100644
--- a/src/backend/utils/adt/pgstatfuncs.c
+++ b/src/backend/utils/adt/pgstatfuncs.c
@@ -494,6 +494,8 @@ pg_stat_get_progress_info(PG_FUNCTION_ARGS)
cmdtype = PROGRESS_COMMAND_BASEBACKUP;
else if (pg_strcasecmp(cmd, "COPY") == 0)
cmdtype = PROGRESS_COMMAND_COPY;
+ else if (pg_strcasecmp(cmd, "CHECKPOINT") == 0)
+ cmdtype = PROGRESS_COMMAND_CHECKPOINT;
else
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -2495,3 +2497,50 @@ pg_stat_get_subscription_worker(PG_FUNCTION_ARGS)
/* Returns the record as Datum */
PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
}
+
+/*
+ * Return checkpoint type (either checkpoint or restartpoint).
+ */
+Datum
+pg_stat_get_progress_checkpoint_type(PG_FUNCTION_ARGS)
+{
+ TimeLineID cur_timeline = (TimeLineID) PG_GETARG_INT64(0);
+
+ if (RecoveryInProgress() || (GetWALInsertionTimeLine() != cur_timeline))
+ PG_RETURN_TEXT_P(CStringGetTextDatum("restartpoint"));
+ else
+ PG_RETURN_TEXT_P(CStringGetTextDatum("checkpoint"));
+}
+
+/*
+ * Return checkpoint kind based on the flags set.
+ */
+Datum
+pg_stat_get_progress_checkpoint_kind(PG_FUNCTION_ARGS)
+{
+ int64 flags = PG_GETARG_INT64(0);
+ char ckpt_kind[MAXPGPATH];
+
+ MemSet(ckpt_kind, 0, MAXPGPATH);
+ snprintf(ckpt_kind, MAXPGPATH, "%s%s%s%s%s%s%s%s%s",
+ (flags == 0) ? "unknown" : "",
+ (flags & CHECKPOINT_IS_SHUTDOWN) ? "shutdown " : "",
+ (flags & CHECKPOINT_END_OF_RECOVERY) ? "end-of-recovery " : "",
+ (flags & CHECKPOINT_IMMEDIATE) ? "immediate " : "",
+ (flags & CHECKPOINT_FORCE) ? "force " : "",
+ (flags & CHECKPOINT_WAIT) ? "wait " : "",
+ (flags & CHECKPOINT_CAUSE_XLOG) ? "wal " : "",
+ (flags & CHECKPOINT_CAUSE_TIME) ? "time " : "",
+ (flags & CHECKPOINT_FLUSH_ALL) ? "flush-all" : "");
+
+ PG_RETURN_TEXT_P(CStringGetTextDatum(ckpt_kind));
+}
+
+/*
+ * Return start time of the checkpoint.
+ */
+Datum
+pg_stat_get_progress_checkpoint_start_time(PG_FUNCTION_ARGS)
+{
+ PG_RETURN_TIMESTAMPTZ(PG_GETARG_INT64(0));
+}
\ No newline at end of file
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7f1ee97f55..06a84e2488 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5353,6 +5353,18 @@
proargmodes => '{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
proargnames => '{cmdtype,pid,datid,relid,param1,param2,param3,param4,param5,param6,param7,param8,param9,param10,param11,param12,param13,param14,param15,param16,param17,param18,param19,param20}',
prosrc => 'pg_stat_get_progress_info' },
+{ oid => '560', descr => 'return checkpoint type',
+ proname => 'pg_stat_get_progress_checkpoint_type', provolatile => 'i',
+ prorettype => 'text', proargtypes => 'int8',
+ prosrc => 'pg_stat_get_progress_checkpoint_type' },
+{ oid => '561', descr => 'return checkpoint kind',
+ proname => 'pg_stat_get_progress_checkpoint_kind', provolatile => 'i',
+ prorettype => 'text', proargtypes => 'int8',
+ prosrc => 'pg_stat_get_progress_checkpoint_kind' },
+{ oid => '562', descr => 'return elapsed time of the checkpoint',
+ proname => 'pg_stat_get_progress_checkpoint_start_time', provolatile => 'i',
+ prorettype => 'timestamptz', proargtypes => 'int8',
+ prosrc => 'pg_stat_get_progress_checkpoint_start_time' },
{ oid => '3099',
descr => 'statistics: information about currently active replication',
proname => 'pg_stat_get_wal_senders', prorows => '10', proisstrict => 'f',
diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h
index a28938caf4..19e9e93072 100644
--- a/src/include/commands/progress.h
+++ b/src/include/commands/progress.h
@@ -151,4 +151,32 @@
#define PROGRESS_COPY_TYPE_PIPE 3
#define PROGRESS_COPY_TYPE_CALLBACK 4
+/* Progress parameters for checkpoint */
+#define PROGRESS_CHECKPOINT_TIMELINE 0
+#define PROGRESS_CHECKPOINT_KIND 1
+#define PROGRESS_CHECKPOINT_LSN 2
+#define PROGRESS_CHECKPOINT_START_TIMESTAMP 3
+#define PROGRESS_CHECKPOINT_PHASE 4
+#define PROGRESS_CHECKPOINT_BUFFERS_TOTAL 5
+#define PROGRESS_CHECKPOINT_BUFFERS_PROCESSED 6
+#define PROGRESS_CHECKPOINT_BUFFERS_WRITTEN 7
+#define PROGRESS_CHECKPOINT_FILES_TOTAL 8
+#define PROGRESS_CHECKPOINT_FILES_SYNCED 9
+
+/* Phases of checkpoint (as advertised via PROGRESS_CHECKPOINT_PHASE) */
+#define PROGRESS_CHECKPOINT_PHASE_INIT 1
+#define PROGRESS_CHECKPOINT_PHASE_REPLI_SLOTS 2
+#define PROGRESS_CHECKPOINT_PHASE_SNAPSHOTS 3
+#define PROGRESS_CHECKPOINT_PHASE_LOGICAL_REWRITE_MAPPINGS 4
+#define PROGRESS_CHECKPOINT_PHASE_CLOG_PAGES 5
+#define PROGRESS_CHECKPOINT_PHASE_COMMITTS_PAGES 6
+#define PROGRESS_CHECKPOINT_PHASE_SUBTRANS_PAGES 7
+#define PROGRESS_CHECKPOINT_PHASE_MULTIXACT_PAGES 8
+#define PROGRESS_CHECKPOINT_PHASE_SLRU_PAGES 9
+#define PROGRESS_CHECKPOINT_PHASE_BUFFERS 10
+#define PROGRESS_CHECKPOINT_PHASE_FILE_SYNC 11
+#define PROGRESS_CHECKPOINT_PHASE_TWO_PHASE 12
+#define PROGRESS_CHECKPOINT_PHASE_OLD_XLOG_RECYCLE 13
+#define PROGRESS_CHECKPOINT_PHASE_FINALIZE 14
+
#endif
diff --git a/src/include/utils/backend_progress.h b/src/include/utils/backend_progress.h
index 47bf8029b0..02d51fb948 100644
--- a/src/include/utils/backend_progress.h
+++ b/src/include/utils/backend_progress.h
@@ -27,7 +27,8 @@ typedef enum ProgressCommandType
PROGRESS_COMMAND_CLUSTER,
PROGRESS_COMMAND_CREATE_INDEX,
PROGRESS_COMMAND_BASEBACKUP,
- PROGRESS_COMMAND_COPY
+ PROGRESS_COMMAND_COPY,
+ PROGRESS_COMMAND_CHECKPOINT
} ProgressCommandType;
#define PGSTAT_NUM_PROGRESS_PARAM 20
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 1420288d67..e3024975d9 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1897,6 +1897,38 @@ pg_stat_progress_basebackup| SELECT s.pid,
s.param4 AS tablespaces_total,
s.param5 AS tablespaces_streamed
FROM pg_stat_get_progress_info('BASEBACKUP'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
+pg_stat_progress_checkpoint| SELECT s.pid,
+ pg_stat_get_progress_checkpoint_type(s.param1) AS type,
+ pg_stat_get_progress_checkpoint_kind(s.param2) AS kind,
+ ('0/0'::pg_lsn + (
+ CASE
+ WHEN (s.param3 < 0) THEN pow((2)::numeric, (64)::numeric)
+ ELSE (0)::numeric
+ END + (s.param3)::numeric)) AS start_lsn,
+ pg_stat_get_progress_checkpoint_start_time(s.param4) AS start_time,
+ CASE s.param5
+ WHEN 1 THEN 'initializing'::text
+ WHEN 2 THEN 'checkpointing replication slots'::text
+ WHEN 3 THEN 'checkpointing snapshots'::text
+ WHEN 4 THEN 'checkpointing logical rewrite mappings'::text
+ WHEN 5 THEN 'checkpointing commit log pages'::text
+ WHEN 6 THEN 'checkpointing commit time stamp pages'::text
+ WHEN 7 THEN 'checkpointing subtransaction pages'::text
+ WHEN 8 THEN 'checkpointing multixact pages'::text
+ WHEN 9 THEN 'checkpointing SLRU pages'::text
+ WHEN 10 THEN 'checkpointing buffers'::text
+ WHEN 11 THEN 'performing sync requests'::text
+ WHEN 12 THEN 'performing two phase checkpoint'::text
+ WHEN 13 THEN 'recycling old XLOG files'::text
+ WHEN 14 THEN 'Finalizing'::text
+ ELSE NULL::text
+ END AS phase,
+ s.param6 AS total_buffer_writes,
+ s.param7 AS buffers_processed,
+ s.param8 AS buffers_written,
+ s.param9 AS total_file_syncs,
+ s.param10 AS files_synced
+ FROM pg_stat_get_progress_info('CHECKPOINT'::text) s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20);
pg_stat_progress_cluster| SELECT s.pid,
s.datid,
d.datname,
--
2.25.1
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-17 11:11 ` Matthias van de Meent <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Matthias van de Meent @ 2022-02-17 11:11 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; PostgreSQL Hackers <[email protected]>
On Thu, 17 Feb 2022 at 07:56, Nitin Jadhav
<[email protected]> wrote:
>
> > Progress parameters are int64, so all of the new 'checkpoint start
> > location' (lsn = uint64), 'triggering backend PID' (int), 'elapsed
> > time' (store as start time in stat_progress, timestamp fits in 64
> > bits) and 'checkpoint or restartpoint?' (boolean) would each fit in a
> > current stat_progress parameter. Some processing would be required at
> > the view, but that's not impossible to overcome.
>
> Thank you for sharing the information. 'triggering backend PID' (int)
> - can be stored without any problem. 'checkpoint or restartpoint?'
> (boolean) - can be stored as a integer value like
> PROGRESS_CHECKPOINT_TYPE_CHECKPOINT(0) and
> PROGRESS_CHECKPOINT_TYPE_RESTARTPOINT(1). 'elapsed time' (store as
> start time in stat_progress, timestamp fits in 64 bits) - As
> Timestamptz is of type int64 internally, so we can store the timestamp
> value in the progres parameter and then expose a function like
> 'pg_stat_get_progress_checkpoint_elapsed' which takes int64 (not
> Timestamptz) as argument and then returns string representing the
> elapsed time.
No need to use a string there; I think exposing the checkpoint start
time is good enough. The conversion of int64 to timestamp[tz] can be
done in SQL (although I'm not sure that exposing the internal bitwise
representation of Interval should be exposed to that extent) [0].
Users can then extract the duration interval using now() - start_time,
which also allows the user to use their own preferred formatting.
> This function can be called in the view. Is it
> safe/advisable to use int64 type here rather than Timestamptz for this
> purpose?
Yes, this must be exposed through int64, as the sql-callable
pg_stat_get_progress_info only exposes bigint columns. Any
transformation function may return other types (see
pg_indexam_progress_phasename for an example of that).
> 'checkpoint start location' (lsn = uint64) - I feel we
> cannot use progress parameters for this case. As assigning uint64 to
> int64 type would be an issue for larger values and can lead to hidden
> bugs.
Not necessarily - we can (without much trouble) do a bitwise cast from
uint64 to int64, and then (in SQL) cast it back to a pg_lsn [1]. Not
very elegant, but it works quite well.
Kind regards,
Matthias van de Meent
[0] Assuming we don't care about the years past 294246 CE (2942467 is
when int64 overflows into negatives), the following works without any
precision losses: SELECT
to_timestamp((stat.my_int64::bigint/1000000)::float8) +
make_interval(0, 0, 0, 0, 0, 0, MOD(stat.my_int64, 1000000)::float8 /
1000000::float8) FROM (SELECT 1::bigint) AS stat(my_int64);
[1] SELECT '0/0'::pg_lsn + ((CASE WHEN stat.my_int64 < 0 THEN
pow(2::numeric, 64::numeric)::numeric ELSE 0::numeric END) +
stat.my_int64::numeric) FROM (SELECT -2::bigint /* 0xFFFFFFFF/FFFFFFFE
*/ AS my_bigint_lsn) AS stat(my_int64);
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
@ 2022-02-18 06:32 ` Nitin Jadhav <[email protected]>
1 sibling, 0 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-18 06:32 UTC (permalink / raw)
To: Matthias van de Meent <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> Interesting idea, and overall a nice addition to the
> pg_stat_progress_* reporting infrastructure.
>
> Could you add your patch to the current commitfest at
> https://commitfest.postgresql.org/37/?
>
> See below for some comments on the patch:
Thanks you for reviewing.
I have added it to the commitfest - https://commitfest.postgresql.org/37/3545/
> > xlog.c @ checkpoint_progress_start, checkpoint_progress_update_param, checkpoint_progress_end
> > + /* In bootstrap mode, we don't actually record anything. */
> > + if (IsBootstrapProcessingMode())
> > + return;
>
> Why do you check against the state of the system?
> pgstat_progress_update_* already provides protections against updating
> the progress tables if the progress infrastructure is not loaded; and
> otherwise (in the happy path) the cost of updating the progress fields
> will be quite a bit higher than normal. Updating stat_progress isn't
> very expensive (quite cheap, really), so I don't quite get why you
> guard against reporting stats when you expect no other client to be
> listening.
Nice point. I agree that the extra guards(IsBootstrapProcessingMode()
and (flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY)) ==
0) are not needed as the progress reporting mechanism handles that
internally (It only updates when there is an access to the
pg_stat_progress_activity view). I am planning to add the progress of
checkpoint during shutdown and end-of-recovery cases in server logs as
we don't have access to the view. In this case these guards are
necessary. checkpoint_progress_update_param() is a generic function to
report progress to the view or server logs. Thoughts?
> I think you can simplify this a lot by directly using
> pgstat_progress_update_param() instead.
>
> > xlog.c @ checkpoint_progress_start
> > + pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
> > + PROGRESS_CHECKPOINT_PHASE_INIT);
> > + if (flags & CHECKPOINT_CAUSE_XLOG)
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> > + PROGRESS_CHECKPOINT_KIND_WAL);
> > + else if (flags & CHECKPOINT_CAUSE_TIME)
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> > + PROGRESS_CHECKPOINT_KIND_TIME);
> > + [...]
>
> Could you assign the kind of checkpoint to a local variable, and then
> update the "phase" and "kind" parameters at the same time through
> pgstat_progress_update_multi_param(2, ...)? See
> BuildRelationExtStatistics in extended_stats.c for an example usage.
I will make use of pgstat_progress_update_multi_param() in the next
patch to replace multiple calls to checkpoint_progress_update_param().
> Note that regardless of whether checkpoint_progress_update* will
> remain, the checks done in that function already have been checked in
> this function as well, so you can use the pgstat_* functions directly.
As I mentioned before I am planning to add progress reporting in the
server logs, checkpoint_progress_update_param() is required and it
makes the job easier.
> > monitoring.sgml
> > + <structname>pg_stat_progress_checkpoint</structname> view will contain a
> > + single row indicating the progress of checkpoint operation.
>
>... add "if a checkpoint is currently active".
I feel adding extra words here to indicate "if a checkpoint is
currently active" is not necessary as the view description provides
that information and also it aligns with the documentation of existing
progress views.
> > + <structfield>total_buffer_writes</structfield> <type>bigint</type>
> > + <structfield>total_file_syncs</structfield> <type>bigint</type>
>
> The other progress tables use [type]_total as column names for counter
> targets (e.g. backup_total for backup_streamed, heap_blks_total for
> heap_blks_scanned, etc.). I think that `buffers_total` and
> `files_total` would be better column names.
I agree and I will update this in the next patch.
> > + The checkpoint operation is requested due to XLOG filling.
>
> + The checkpoint was started because >max_wal_size< of WAL was written.
How about this "The checkpoint is started because max_wal_size is reached".
> > + The checkpoint operation is requested due to timeout.
>
> + The checkpoint was started due to the expiration of a
> >checkpoint_timeout< interval
"The checkpoint is started because checkpoint_timeout expired".
> > + The checkpoint operation is forced even if no XLOG activity has occurred
> > + since the last one.
>
> + Some operation forced a checkpoint.
"The checkpoint is started because some operation forced a checkpoint".
> > + <entry><literal>checkpointing CommitTs pages</literal></entry>
>
> CommitTs -> Commit time stamp
I will handle this in the next patch.
Thanks & Regards,
Nitin Jadhav
> On Thu, 10 Feb 2022 at 07:53, Nitin Jadhav
> <[email protected]> wrote:
> >
> > > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > > >
> > > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> > >
> > > I agree to provide above mentioned information as part of showing the
> > > progress of current checkpoint operation. I am currently looking into
> > > the code to know if any other information can be added.
> >
> > Here is the initial patch to show the progress of checkpoint through
> > pg_stat_progress_checkpoint view. Please find the attachment.
> >
> > The information added to this view are pid - process ID of a
> > CHECKPOINTER process, kind - kind of checkpoint indicates the reason
> > for checkpoint (values can be wal, time or force), phase - indicates
> > the current phase of checkpoint operation, total_buffer_writes - total
> > number of buffers to be written, buffers_processed - number of buffers
> > processed, buffers_written - number of buffers written,
> > total_file_syncs - total number of files to be synced, files_synced -
> > number of files synced.
> >
> > There are many operations happen as part of checkpoint. For each of
> > the operation I am updating the phase field of
> > pg_stat_progress_checkpoint view. The values supported for this field
> > are initializing, checkpointing replication slots, checkpointing
> > snapshots, checkpointing logical rewrite mappings, checkpointing CLOG
> > pages, checkpointing CommitTs pages, checkpointing SUBTRANS pages,
> > checkpointing MULTIXACT pages, checkpointing SLRU pages, checkpointing
> > buffers, performing sync requests, performing two phase checkpoint,
> > recycling old XLOG files and Finalizing. In case of checkpointing
> > buffers phase, the fields total_buffer_writes, buffers_processed and
> > buffers_written shows the detailed progress of writing buffers. In
> > case of performing sync requests phase, the fields total_file_syncs
> > and files_synced shows the detailed progress of syncing files. In
> > other phases, only the phase field is getting updated and it is
> > difficult to show the progress because we do not get the total number
> > of files count without traversing the directory. It is not worth to
> > calculate that as it affects the performance of the checkpoint. I also
> > gave a thought to just mention the number of files processed, but this
> > wont give a meaningful progress information (It can be treated as
> > statistics). Hence just updating the phase field in those scenarios.
> >
> > Apart from above fields, I am planning to add few more fields to the
> > view in the next patch. That is, process ID of the backend process
> > which triggered a CHECKPOINT command, checkpoint start location, filed
> > to indicate whether it is a checkpoint or restartpoint and elapsed
> > time of the checkpoint operation. Please share your thoughts. I would
> > be happy to add any other information that contributes to showing the
> > progress of checkpoint.
> >
> > As per the discussion in this thread, there should be some mechanism
> > to show the progress of checkpoint during shutdown and end-of-recovery
> > cases as we cannot access pg_stat_progress_checkpoint in those cases.
> > I am working on this to use log_startup_progress_interval mechanism to
> > log the progress in the server logs.
> >
> > Kindly review the patch and share your thoughts.
>
> Interesting idea, and overall a nice addition to the
> pg_stat_progress_* reporting infrastructure.
>
> Could you add your patch to the current commitfest at
> https://commitfest.postgresql.org/37/?
>
> See below for some comments on the patch:
>
> > xlog.c @ checkpoint_progress_start, checkpoint_progress_update_param, checkpoint_progress_end
> > + /* In bootstrap mode, we don't actually record anything. */
> > + if (IsBootstrapProcessingMode())
> > + return;
>
> Why do you check against the state of the system?
> pgstat_progress_update_* already provides protections against updating
> the progress tables if the progress infrastructure is not loaded; and
> otherwise (in the happy path) the cost of updating the progress fields
> will be quite a bit higher than normal. Updating stat_progress isn't
> very expensive (quite cheap, really), so I don't quite get why you
> guard against reporting stats when you expect no other client to be
> listening.
>
> I think you can simplify this a lot by directly using
> pgstat_progress_update_param() instead.
>
> > xlog.c @ checkpoint_progress_start
> > + pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT, InvalidOid);
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_PHASE,
> > + PROGRESS_CHECKPOINT_PHASE_INIT);
> > + if (flags & CHECKPOINT_CAUSE_XLOG)
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> > + PROGRESS_CHECKPOINT_KIND_WAL);
> > + else if (flags & CHECKPOINT_CAUSE_TIME)
> > + checkpoint_progress_update_param(flags, PROGRESS_CHECKPOINT_KIND,
> > + PROGRESS_CHECKPOINT_KIND_TIME);
> > + [...]
>
> Could you assign the kind of checkpoint to a local variable, and then
> update the "phase" and "kind" parameters at the same time through
> pgstat_progress_update_multi_param(2, ...)? See
> BuildRelationExtStatistics in extended_stats.c for an example usage.
> Note that regardless of whether checkpoint_progress_update* will
> remain, the checks done in that function already have been checked in
> this function as well, so you can use the pgstat_* functions directly.
>
> > monitoring.sgml
> > + <structname>pg_stat_progress_checkpoint</structname> view will contain a
> > + single row indicating the progress of checkpoint operation.
>
> ... add "if a checkpoint is currently active".
>
> > + <structfield>total_buffer_writes</structfield> <type>bigint</type>
> > + <structfield>total_file_syncs</structfield> <type>bigint</type>
>
> The other progress tables use [type]_total as column names for counter
> targets (e.g. backup_total for backup_streamed, heap_blks_total for
> heap_blks_scanned, etc.). I think that `buffers_total` and
> `files_total` would be better column names.
>
> > + The checkpoint operation is requested due to XLOG filling.
>
> + The checkpoint was started because >max_wal_size< of WAL was written.
>
> > + The checkpoint operation is requested due to timeout.
>
> + The checkpoint was started due to the expiration of a
> >checkpoint_timeout< interval
>
> > + The checkpoint operation is forced even if no XLOG activity has occurred
> > + since the last one.
>
> + Some operation forced a checkpoint.
>
> > + <entry><literal>checkpointing CommitTs pages</literal></entry>
>
> CommitTs -> Commit time stamp
>
> Thanks for working on this.
>
> Kind regards,
>
> Matthias van de Meent
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
@ 2022-02-22 14:40 ` Ashutosh Sharma <[email protected]>
2022-02-23 13:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2 siblings, 1 reply; 85+ messages in thread
From: Ashutosh Sharma @ 2022-02-22 14:40 UTC (permalink / raw)
To: Nitin Jadhav <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
+/* Kinds of checkpoint (as advertised via PROGRESS_CHECKPOINT_KIND) */
+#define PROGRESS_CHECKPOINT_KIND_WAL 0
+#define PROGRESS_CHECKPOINT_KIND_TIME 1
+#define PROGRESS_CHECKPOINT_KIND_FORCE 2
+#define PROGRESS_CHECKPOINT_KIND_UNKNOWN 3
On what basis have you classified the above into the various types of
checkpoints? AFAIK, the first two types are based on what triggered
the checkpoint (whether it was the checkpoint_timeout or maz_wal_size
settings) while the third type indicates the force checkpoint that can
happen when the checkpoint is triggered for various reasons e.g. .
during createb or dropdb etc. This is quite possible that both the
PROGRESS_CHECKPOINT_KIND_TIME and PROGRESS_CHECKPOINT_KIND_FORCE flags
are set for the checkpoint because multiple checkpoint requests are
processed at one go, so what type of checkpoint would that be?
+ */
+ if ((flags & (CHECKPOINT_IS_SHUTDOWN |
CHECKPOINT_END_OF_RECOVERY)) == 0)
+ {
+
pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT,
InvalidOid);
+ checkpoint_progress_update_param(flags,
PROGRESS_CHECKPOINT_PHASE,
+
PROGRESS_CHECKPOINT_PHASE_INIT);
+ if (flags & CHECKPOINT_CAUSE_XLOG)
+ checkpoint_progress_update_param(flags,
PROGRESS_CHECKPOINT_KIND,
+
PROGRESS_CHECKPOINT_KIND_WAL);
+ else if (flags & CHECKPOINT_CAUSE_TIME)
+ checkpoint_progress_update_param(flags,
PROGRESS_CHECKPOINT_KIND,
+
PROGRESS_CHECKPOINT_KIND_TIME);
+ else if (flags & CHECKPOINT_FORCE)
+ checkpoint_progress_update_param(flags,
PROGRESS_CHECKPOINT_KIND,
+
PROGRESS_CHECKPOINT_KIND_FORCE);
+ else
+ checkpoint_progress_update_param(flags,
PROGRESS_CHECKPOINT_KIND,
+
PROGRESS_CHECKPOINT_KIND_UNKNOWN);
+ }
+}
--
With Regards,
Ashutosh Sharma.
On Thu, Feb 10, 2022 at 12:23 PM Nitin Jadhav
<[email protected]> wrote:
>
> > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > >
> > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> >
> > I agree to provide above mentioned information as part of showing the
> > progress of current checkpoint operation. I am currently looking into
> > the code to know if any other information can be added.
>
> Here is the initial patch to show the progress of checkpoint through
> pg_stat_progress_checkpoint view. Please find the attachment.
>
> The information added to this view are pid - process ID of a
> CHECKPOINTER process, kind - kind of checkpoint indicates the reason
> for checkpoint (values can be wal, time or force), phase - indicates
> the current phase of checkpoint operation, total_buffer_writes - total
> number of buffers to be written, buffers_processed - number of buffers
> processed, buffers_written - number of buffers written,
> total_file_syncs - total number of files to be synced, files_synced -
> number of files synced.
>
> There are many operations happen as part of checkpoint. For each of
> the operation I am updating the phase field of
> pg_stat_progress_checkpoint view. The values supported for this field
> are initializing, checkpointing replication slots, checkpointing
> snapshots, checkpointing logical rewrite mappings, checkpointing CLOG
> pages, checkpointing CommitTs pages, checkpointing SUBTRANS pages,
> checkpointing MULTIXACT pages, checkpointing SLRU pages, checkpointing
> buffers, performing sync requests, performing two phase checkpoint,
> recycling old XLOG files and Finalizing. In case of checkpointing
> buffers phase, the fields total_buffer_writes, buffers_processed and
> buffers_written shows the detailed progress of writing buffers. In
> case of performing sync requests phase, the fields total_file_syncs
> and files_synced shows the detailed progress of syncing files. In
> other phases, only the phase field is getting updated and it is
> difficult to show the progress because we do not get the total number
> of files count without traversing the directory. It is not worth to
> calculate that as it affects the performance of the checkpoint. I also
> gave a thought to just mention the number of files processed, but this
> wont give a meaningful progress information (It can be treated as
> statistics). Hence just updating the phase field in those scenarios.
>
> Apart from above fields, I am planning to add few more fields to the
> view in the next patch. That is, process ID of the backend process
> which triggered a CHECKPOINT command, checkpoint start location, filed
> to indicate whether it is a checkpoint or restartpoint and elapsed
> time of the checkpoint operation. Please share your thoughts. I would
> be happy to add any other information that contributes to showing the
> progress of checkpoint.
>
> As per the discussion in this thread, there should be some mechanism
> to show the progress of checkpoint during shutdown and end-of-recovery
> cases as we cannot access pg_stat_progress_checkpoint in those cases.
> I am working on this to use log_startup_progress_interval mechanism to
> log the progress in the server logs.
>
> Kindly review the patch and share your thoughts.
>
>
> On Fri, Jan 28, 2022 at 12:24 PM Bharath Rupireddy
> <[email protected]> wrote:
> >
> > On Fri, Jan 21, 2022 at 11:07 AM Nitin Jadhav
> > <[email protected]> wrote:
> > >
> > > > I think the right choice to solve the *general* problem is the
> > > > mentioned pg_stat_progress_checkpoints.
> > > >
> > > > We may want to *additionally* have the ability to log the progress
> > > > specifically for the special cases when we're not able to use that
> > > > view. And in those case, we can perhaps just use the existing
> > > > log_startup_progress_interval parameter for this as well -- at least
> > > > for the startup checkpoint.
> > >
> > > +1
> > >
> > > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > > >
> > > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> > >
> > > I agree to provide above mentioned information as part of showing the
> > > progress of current checkpoint operation. I am currently looking into
> > > the code to know if any other information can be added.
> >
> > As suggested in the other thread by Julien, I'm changing the subject
> > of this thread to reflect the discussion.
> >
> > Regards,
> > Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs)
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-22 14:40 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
@ 2022-02-23 13:37 ` Nitin Jadhav <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Nitin Jadhav @ 2022-02-23 13:37 UTC (permalink / raw)
To: Ashutosh Sharma <[email protected]>; +Cc: Bharath Rupireddy <[email protected]>; Bruce Momjian <[email protected]>; Tom Lane <[email protected]>; Magnus Hagander <[email protected]>; PostgreSQL Hackers <[email protected]>
> On what basis have you classified the above into the various types of
> checkpoints? AFAIK, the first two types are based on what triggered
> the checkpoint (whether it was the checkpoint_timeout or maz_wal_size
> settings) while the third type indicates the force checkpoint that can
> happen when the checkpoint is triggered for various reasons e.g. .
> during createb or dropdb etc. This is quite possible that both the
> PROGRESS_CHECKPOINT_KIND_TIME and PROGRESS_CHECKPOINT_KIND_FORCE flags
> are set for the checkpoint because multiple checkpoint requests are
> processed at one go, so what type of checkpoint would that be?
My initial understanding was wrong. In the v2 patch I have supported
all values for checkpoint kinds and displaying a string in the
pg_stat_progress_checkpoint view which describes all the bits set in
the checkpoint flags.
On Tue, Feb 22, 2022 at 8:10 PM Ashutosh Sharma <[email protected]> wrote:
>
> +/* Kinds of checkpoint (as advertised via PROGRESS_CHECKPOINT_KIND) */
> +#define PROGRESS_CHECKPOINT_KIND_WAL 0
> +#define PROGRESS_CHECKPOINT_KIND_TIME 1
> +#define PROGRESS_CHECKPOINT_KIND_FORCE 2
> +#define PROGRESS_CHECKPOINT_KIND_UNKNOWN 3
>
> On what basis have you classified the above into the various types of
> checkpoints? AFAIK, the first two types are based on what triggered
> the checkpoint (whether it was the checkpoint_timeout or maz_wal_size
> settings) while the third type indicates the force checkpoint that can
> happen when the checkpoint is triggered for various reasons e.g. .
> during createb or dropdb etc. This is quite possible that both the
> PROGRESS_CHECKPOINT_KIND_TIME and PROGRESS_CHECKPOINT_KIND_FORCE flags
> are set for the checkpoint because multiple checkpoint requests are
> processed at one go, so what type of checkpoint would that be?
>
> + */
> + if ((flags & (CHECKPOINT_IS_SHUTDOWN |
> CHECKPOINT_END_OF_RECOVERY)) == 0)
> + {
> +
> pgstat_progress_start_command(PROGRESS_COMMAND_CHECKPOINT,
> InvalidOid);
> + checkpoint_progress_update_param(flags,
> PROGRESS_CHECKPOINT_PHASE,
> +
> PROGRESS_CHECKPOINT_PHASE_INIT);
> + if (flags & CHECKPOINT_CAUSE_XLOG)
> + checkpoint_progress_update_param(flags,
> PROGRESS_CHECKPOINT_KIND,
> +
> PROGRESS_CHECKPOINT_KIND_WAL);
> + else if (flags & CHECKPOINT_CAUSE_TIME)
> + checkpoint_progress_update_param(flags,
> PROGRESS_CHECKPOINT_KIND,
> +
> PROGRESS_CHECKPOINT_KIND_TIME);
> + else if (flags & CHECKPOINT_FORCE)
> + checkpoint_progress_update_param(flags,
> PROGRESS_CHECKPOINT_KIND,
> +
> PROGRESS_CHECKPOINT_KIND_FORCE);
> + else
> + checkpoint_progress_update_param(flags,
> PROGRESS_CHECKPOINT_KIND,
> +
> PROGRESS_CHECKPOINT_KIND_UNKNOWN);
> + }
> +}
>
> --
> With Regards,
> Ashutosh Sharma.
>
> On Thu, Feb 10, 2022 at 12:23 PM Nitin Jadhav
> <[email protected]> wrote:
> >
> > > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > > >
> > > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> > >
> > > I agree to provide above mentioned information as part of showing the
> > > progress of current checkpoint operation. I am currently looking into
> > > the code to know if any other information can be added.
> >
> > Here is the initial patch to show the progress of checkpoint through
> > pg_stat_progress_checkpoint view. Please find the attachment.
> >
> > The information added to this view are pid - process ID of a
> > CHECKPOINTER process, kind - kind of checkpoint indicates the reason
> > for checkpoint (values can be wal, time or force), phase - indicates
> > the current phase of checkpoint operation, total_buffer_writes - total
> > number of buffers to be written, buffers_processed - number of buffers
> > processed, buffers_written - number of buffers written,
> > total_file_syncs - total number of files to be synced, files_synced -
> > number of files synced.
> >
> > There are many operations happen as part of checkpoint. For each of
> > the operation I am updating the phase field of
> > pg_stat_progress_checkpoint view. The values supported for this field
> > are initializing, checkpointing replication slots, checkpointing
> > snapshots, checkpointing logical rewrite mappings, checkpointing CLOG
> > pages, checkpointing CommitTs pages, checkpointing SUBTRANS pages,
> > checkpointing MULTIXACT pages, checkpointing SLRU pages, checkpointing
> > buffers, performing sync requests, performing two phase checkpoint,
> > recycling old XLOG files and Finalizing. In case of checkpointing
> > buffers phase, the fields total_buffer_writes, buffers_processed and
> > buffers_written shows the detailed progress of writing buffers. In
> > case of performing sync requests phase, the fields total_file_syncs
> > and files_synced shows the detailed progress of syncing files. In
> > other phases, only the phase field is getting updated and it is
> > difficult to show the progress because we do not get the total number
> > of files count without traversing the directory. It is not worth to
> > calculate that as it affects the performance of the checkpoint. I also
> > gave a thought to just mention the number of files processed, but this
> > wont give a meaningful progress information (It can be treated as
> > statistics). Hence just updating the phase field in those scenarios.
> >
> > Apart from above fields, I am planning to add few more fields to the
> > view in the next patch. That is, process ID of the backend process
> > which triggered a CHECKPOINT command, checkpoint start location, filed
> > to indicate whether it is a checkpoint or restartpoint and elapsed
> > time of the checkpoint operation. Please share your thoughts. I would
> > be happy to add any other information that contributes to showing the
> > progress of checkpoint.
> >
> > As per the discussion in this thread, there should be some mechanism
> > to show the progress of checkpoint during shutdown and end-of-recovery
> > cases as we cannot access pg_stat_progress_checkpoint in those cases.
> > I am working on this to use log_startup_progress_interval mechanism to
> > log the progress in the server logs.
> >
> > Kindly review the patch and share your thoughts.
> >
> >
> > On Fri, Jan 28, 2022 at 12:24 PM Bharath Rupireddy
> > <[email protected]> wrote:
> > >
> > > On Fri, Jan 21, 2022 at 11:07 AM Nitin Jadhav
> > > <[email protected]> wrote:
> > > >
> > > > > I think the right choice to solve the *general* problem is the
> > > > > mentioned pg_stat_progress_checkpoints.
> > > > >
> > > > > We may want to *additionally* have the ability to log the progress
> > > > > specifically for the special cases when we're not able to use that
> > > > > view. And in those case, we can perhaps just use the existing
> > > > > log_startup_progress_interval parameter for this as well -- at least
> > > > > for the startup checkpoint.
> > > >
> > > > +1
> > > >
> > > > > We need at least a trace of the number of buffers to sync (num_to_scan) before the checkpoint start, instead of just emitting the stats at the end.
> > > > >
> > > > > Bharat, it would be good to show the buffers synced counter and the total buffers to sync, checkpointer pid, substep it is running, whether it is on target for completion, checkpoint_Reason
> > > > > (manual/times/forced). BufferSync has several variables tracking the sync progress locally, and we may need some refactoring here.
> > > >
> > > > I agree to provide above mentioned information as part of showing the
> > > > progress of current checkpoint operation. I am currently looking into
> > > > the code to know if any other information can be added.
> > >
> > > As suggested in the other thread by Julien, I'm changing the subject
> > > of this thread to reflect the discussion.
> > >
> > > Regards,
> > > Bharath Rupireddy.
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH v26 8/9] Row pattern recognition patch (typedefs.list).
@ 2024-12-30 12:44 Tatsuo Ishii <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Tatsuo Ishii @ 2024-12-30 12:44 UTC (permalink / raw)
---
src/tools/pgindent/typedefs.list | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e1c4f913f8..72906d0fc1 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -1691,6 +1691,7 @@ NamedLWLockTrancheRequest
NamedTuplestoreScan
NamedTuplestoreScanState
NamespaceInfo
+NavigationInfo
NestLoop
NestLoopParam
NestLoopState
@@ -2317,6 +2318,9 @@ RI_CompareKey
RI_ConstraintInfo
RI_QueryHashEntry
RI_QueryKey
+RPCommonSyntax
+RPSkipTo
+RPSubsetItem
RTEKind
RTEPermissionInfo
RWConflict
@@ -2673,6 +2677,7 @@ SimpleStringList
SimpleStringListCell
SingleBoundSortItem
Size
+SkipContext
SkipPages
SlabBlock
SlabContext
@@ -2764,6 +2769,7 @@ StreamStopReason
String
StringInfo
StringInfoData
+StringSet
StripnullState
SubLink
SubLinkType
@@ -3088,6 +3094,7 @@ VarString
VarStringSortSupport
Variable
VariableAssignHook
+VariablePos
VariableSetKind
VariableSetStmt
VariableShowStmt
--
2.25.1
----Next_Part(Mon_Dec_30_22_37_18_2024_171)--
Content-Type: Text/X-Patch; charset=us-ascii
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
filename="v26-0009-Allow-to-print-raw-parse-tree.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH 1/3] stress test for repack concurrently
@ 2025-12-13 17:13 Mikhail Nikalayeu <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Mikhail Nikalayeu @ 2025-12-13 17:13 UTC (permalink / raw)
---
contrib/amcheck/meson.build | 1 +
contrib/amcheck/t/007_repack_1.pl | 121 ++++++++++++++++++++++++++++++
doc/src/sgml/regress.sgml | 16 ++++
3 files changed, 138 insertions(+)
create mode 100644 contrib/amcheck/t/007_repack_1.pl
diff --git a/contrib/amcheck/meson.build b/contrib/amcheck/meson.build
index d5137ef691d..cb4bc32e98a 100644
--- a/contrib/amcheck/meson.build
+++ b/contrib/amcheck/meson.build
@@ -50,6 +50,7 @@ tests += {
't/004_verify_nbtree_unique.pl',
't/005_pitr.pl',
't/006_verify_gin.pl',
+ 't/007_repack_1.pl',
],
},
}
diff --git a/contrib/amcheck/t/007_repack_1.pl b/contrib/amcheck/t/007_repack_1.pl
new file mode 100644
index 00000000000..17f3caa3a38
--- /dev/null
+++ b/contrib/amcheck/t/007_repack_1.pl
@@ -0,0 +1,121 @@
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+
+# Test REPACK CONCURRENTLY with concurrent modifications
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+
+use Test::More;
+
+# PG_TEST_EXTRA carries a numerical stress value; 0 disables the test,
+# 1 means the test takes about 6 seconds, and the increase should be
+# roughly linear.
+my $matched = ($ENV{PG_TEST_EXTRA} // '') =~ /\bstress_concurrently(?:=(?<stressval>[0-9]*))?\b/;
+my $stressval = $+{stressval} // 1;
+if (!$matched or $stressval == 0)
+{
+ plan skip_all => 'skipping disabled REPACK CONCURRENTLY stress test';
+}
+
+note "stressval is $stressval";
+
+my $node;
+
+# Test set-up
+$node = PostgreSQL::Test::Cluster->new('CIC_test');
+$node->init;
+$node->append_conf('postgresql.conf',
+ 'lock_timeout = ' . (1000 * $PostgreSQL::Test::Utils::timeout_default));
+$node->append_conf(
+ 'postgresql.conf', qq(
+wal_level = logical
+));
+
+my $nrows = 10_000 * $stressval;
+my $duration = 6 * $stressval;
+my $no_hot = int(rand(2));
+
+$node->start;
+$node->safe_psql('postgres', q(CREATE TABLE tbl(id int PRIMARY KEY, val int)));
+
+if ($no_hot)
+{
+ $node->safe_psql('postgres', q(CREATE INDEX test_idx ON tbl(val);));
+}
+else
+{
+ $node->safe_psql('postgres', q(CREATE INDEX test_idx ON tbl(id);));
+}
+
+
+# Load amcheck
+$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
+
+# Insert $nrows rows into tbl
+$node->safe_psql('postgres', qq(
+ INSERT INTO tbl SELECT g, g FROM generate_series(1, $nrows) g
+));
+
+my $sum = $node->safe_psql('postgres', q(
+ SELECT SUM(val) AS sum FROM tbl
+));
+
+
+$node->pgbench(
+"--no-vacuum --client=30 --jobs=4 --exit-on-abort -T $duration",
+0,
+[qr{actually processed}],
+[qr{^$}],
+'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY',
+{
+ 'concurrent_ops' => qq(
+ SELECT pg_try_advisory_lock(42)::integer AS gotlock \\gset
+ \\if :gotlock
+ REPACK (CONCURRENTLY) tbl USING INDEX tbl_pkey;
+ SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true);
+ SELECT bt_index_parent_check('test_idx', heapallindexed => true);
+ \\sleep 10 ms
+
+ REPACK (CONCURRENTLY) tbl USING INDEX test_idx;
+ SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true);
+ SELECT bt_index_parent_check('test_idx', heapallindexed => true);
+ \\sleep 10 ms
+
+ REPACK (CONCURRENTLY) tbl;
+ SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true);
+ SELECT bt_index_parent_check('test_idx', heapallindexed => true);
+ \\sleep 10 ms
+
+ SELECT pg_advisory_unlock(42);
+ \\else
+ \\set num_a random(1, $nrows)
+ \\set num_b random(1, $nrows)
+ \\set diff random(1, 10000)
+ BEGIN;
+ UPDATE tbl SET val = val + :diff WHERE id = :num_a;
+ \\sleep 1 ms
+ UPDATE tbl SET val = val - :diff WHERE id = :num_b;
+ \\sleep 1 ms
+ COMMIT;
+
+ BEGIN
+ --TRANSACTION ISOLATION LEVEL REPEATABLE READ
+ ;
+ SELECT 1;
+ \\sleep 1 ms
+ SELECT COALESCE(SUM(val), 0) AS sum FROM tbl \\gset p_
+ \\if :p_sum != $sum
+ COMMIT;
+ SELECT (:p_sum) / 0;
+ \\endif
+
+ COMMIT;
+ \\endif
+ )
+});
+
+$node->stop;
+
+done_testing();
diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml
index c74941bfbf2..c5b1b79a60c 100644
--- a/doc/src/sgml/regress.sgml
+++ b/doc/src/sgml/regress.sgml
@@ -386,6 +386,22 @@ make check-world PG_TEST_EXTRA='kerberos ldap ssl load_balance libpq_encryption'
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>stress_concurrently</literal><optional>=VALUE</optional></term>
+ <listitem>
+ <para>
+ Run some additional, moderately expensive tests for commands with a
+ <literal>CONCURRENTLY</literal> option. Optionally, an integer can be
+ given after an equal sign, which affects how long each such test
+ runs for.
+ Each test is calibrated so that, when given a value of 10,
+ it runs for approximately 60 seconds.
+ If no value is given, 1 is assumed.
+ Explicitly giving a value of 0 causes the test to be skipped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><literal>wal_consistency_checking</literal></term>
<listitem>
--
2.47.3
--tbdehtstrqwksfdh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="0002-one-more-stress-test-for-repack-concurrently.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
* [PATCH 1/3] stress test for repack concurrently
@ 2025-12-13 17:13 Mikhail Nikalayeu <[email protected]>
0 siblings, 0 replies; 85+ messages in thread
From: Mikhail Nikalayeu @ 2025-12-13 17:13 UTC (permalink / raw)
---
contrib/amcheck/meson.build | 1 +
contrib/amcheck/t/007_repack_1.pl | 121 ++++++++++++++++++++++++++++++
doc/src/sgml/regress.sgml | 16 ++++
3 files changed, 138 insertions(+)
create mode 100644 contrib/amcheck/t/007_repack_1.pl
diff --git a/contrib/amcheck/meson.build b/contrib/amcheck/meson.build
index d5137ef691d..cb4bc32e98a 100644
--- a/contrib/amcheck/meson.build
+++ b/contrib/amcheck/meson.build
@@ -50,6 +50,7 @@ tests += {
't/004_verify_nbtree_unique.pl',
't/005_pitr.pl',
't/006_verify_gin.pl',
+ 't/007_repack_1.pl',
],
},
}
diff --git a/contrib/amcheck/t/007_repack_1.pl b/contrib/amcheck/t/007_repack_1.pl
new file mode 100644
index 00000000000..17f3caa3a38
--- /dev/null
+++ b/contrib/amcheck/t/007_repack_1.pl
@@ -0,0 +1,121 @@
+# Copyright (c) 2021-2026, PostgreSQL Global Development Group
+
+# Test REPACK CONCURRENTLY with concurrent modifications
+use strict;
+use warnings FATAL => 'all';
+
+use PostgreSQL::Test::Cluster;
+use PostgreSQL::Test::Utils;
+
+use Test::More;
+
+# PG_TEST_EXTRA carries a numerical stress value; 0 disables the test,
+# 1 means the test takes about 6 seconds, and the increase should be
+# roughly linear.
+my $matched = ($ENV{PG_TEST_EXTRA} // '') =~ /\bstress_concurrently(?:=(?<stressval>[0-9]*))?\b/;
+my $stressval = $+{stressval} // 1;
+if (!$matched or $stressval == 0)
+{
+ plan skip_all => 'skipping disabled REPACK CONCURRENTLY stress test';
+}
+
+note "stressval is $stressval";
+
+my $node;
+
+# Test set-up
+$node = PostgreSQL::Test::Cluster->new('CIC_test');
+$node->init;
+$node->append_conf('postgresql.conf',
+ 'lock_timeout = ' . (1000 * $PostgreSQL::Test::Utils::timeout_default));
+$node->append_conf(
+ 'postgresql.conf', qq(
+wal_level = logical
+));
+
+my $nrows = 10_000 * $stressval;
+my $duration = 6 * $stressval;
+my $no_hot = int(rand(2));
+
+$node->start;
+$node->safe_psql('postgres', q(CREATE TABLE tbl(id int PRIMARY KEY, val int)));
+
+if ($no_hot)
+{
+ $node->safe_psql('postgres', q(CREATE INDEX test_idx ON tbl(val);));
+}
+else
+{
+ $node->safe_psql('postgres', q(CREATE INDEX test_idx ON tbl(id);));
+}
+
+
+# Load amcheck
+$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
+
+# Insert $nrows rows into tbl
+$node->safe_psql('postgres', qq(
+ INSERT INTO tbl SELECT g, g FROM generate_series(1, $nrows) g
+));
+
+my $sum = $node->safe_psql('postgres', q(
+ SELECT SUM(val) AS sum FROM tbl
+));
+
+
+$node->pgbench(
+"--no-vacuum --client=30 --jobs=4 --exit-on-abort -T $duration",
+0,
+[qr{actually processed}],
+[qr{^$}],
+'concurrent operations with REINDEX/CREATE INDEX CONCURRENTLY',
+{
+ 'concurrent_ops' => qq(
+ SELECT pg_try_advisory_lock(42)::integer AS gotlock \\gset
+ \\if :gotlock
+ REPACK (CONCURRENTLY) tbl USING INDEX tbl_pkey;
+ SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true);
+ SELECT bt_index_parent_check('test_idx', heapallindexed => true);
+ \\sleep 10 ms
+
+ REPACK (CONCURRENTLY) tbl USING INDEX test_idx;
+ SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true);
+ SELECT bt_index_parent_check('test_idx', heapallindexed => true);
+ \\sleep 10 ms
+
+ REPACK (CONCURRENTLY) tbl;
+ SELECT bt_index_parent_check('tbl_pkey', heapallindexed => true);
+ SELECT bt_index_parent_check('test_idx', heapallindexed => true);
+ \\sleep 10 ms
+
+ SELECT pg_advisory_unlock(42);
+ \\else
+ \\set num_a random(1, $nrows)
+ \\set num_b random(1, $nrows)
+ \\set diff random(1, 10000)
+ BEGIN;
+ UPDATE tbl SET val = val + :diff WHERE id = :num_a;
+ \\sleep 1 ms
+ UPDATE tbl SET val = val - :diff WHERE id = :num_b;
+ \\sleep 1 ms
+ COMMIT;
+
+ BEGIN
+ --TRANSACTION ISOLATION LEVEL REPEATABLE READ
+ ;
+ SELECT 1;
+ \\sleep 1 ms
+ SELECT COALESCE(SUM(val), 0) AS sum FROM tbl \\gset p_
+ \\if :p_sum != $sum
+ COMMIT;
+ SELECT (:p_sum) / 0;
+ \\endif
+
+ COMMIT;
+ \\endif
+ )
+});
+
+$node->stop;
+
+done_testing();
diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml
index c74941bfbf2..c5b1b79a60c 100644
--- a/doc/src/sgml/regress.sgml
+++ b/doc/src/sgml/regress.sgml
@@ -386,6 +386,22 @@ make check-world PG_TEST_EXTRA='kerberos ldap ssl load_balance libpq_encryption'
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><literal>stress_concurrently</literal><optional>=VALUE</optional></term>
+ <listitem>
+ <para>
+ Run some additional, moderately expensive tests for commands with a
+ <literal>CONCURRENTLY</literal> option. Optionally, an integer can be
+ given after an equal sign, which affects how long each such test
+ runs for.
+ Each test is calibrated so that, when given a value of 10,
+ it runs for approximately 60 seconds.
+ If no value is given, 1 is assumed.
+ Explicitly giving a value of 0 causes the test to be skipped.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><literal>wal_consistency_checking</literal></term>
<listitem>
--
2.47.3
--tbdehtstrqwksfdh
Content-Type: text/x-diff; charset=utf-8
Content-Disposition: attachment;
filename="0002-one-more-stress-test-for-repack-concurrently.patch"
^ permalink raw reply [nested|flat] 85+ messages in thread
end of thread, other threads:[~2025-12-13 17:13 UTC | newest]
Thread overview: 85+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2019-03-26 13:31 [PATCH v10] Generated columns Peter Eisentraut <[email protected]>
2019-09-05 11:21 [PATCH v7 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v12 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v7 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v8 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v9 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v6 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v11 1/4] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2019-09-05 11:21 [PATCH v5 1/3] Move callback-call from ReadPageInternal to XLogReadRecord. Kyotaro Horiguchi <[email protected]>
2022-01-21 05:37 Re: Report checkpoint progress in server logs Nitin Jadhav <[email protected]>
2022-01-28 06:54 ` Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-10 06:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-15 12:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-16 19:51 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-16 20:02 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-17 06:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:05 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-17 17:09 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 17:27 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 06:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-18 07:43 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-18 14:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-19 05:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-22 06:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-22 18:42 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-23 14:23 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 19:50 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-23 13:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 16:16 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-25 15:07 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-27 15:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 04:51 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 06:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-28 12:33 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-02-28 13:10 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-03-01 08:57 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 11:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-02 18:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-03-07 14:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 14:55 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-19 00:15 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-04-05 09:45 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Michael Paquier <[email protected]>
2022-06-13 13:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-04-08 14:52 ` Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint) Matthias van de Meent <[email protected]>
2022-04-08 15:20 ` Re: Size of pg_rewrite Dagfinn Ilmari Mannsåker <[email protected]>
2022-04-08 16:09 ` Re: Size of pg_rewrite (Was: Report checkpoint progress with pg_stat_progress_checkpoint) Andres Freund <[email protected]>
2022-06-13 13:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-07-07 00:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-07-28 09:38 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-04 08:25 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Drouvot, Bertrand <[email protected]>
2022-11-15 11:41 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-11-15 20:04 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 10:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-16 18:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-16 19:19 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-16 19:52 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-17 14:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-17 16:24 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-11-17 17:18 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Tom Lane <[email protected]>
2022-11-17 17:21 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Robert Haas <[email protected]>
2022-11-17 13:31 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Bharath Rupireddy <[email protected]>
2022-11-15 20:18 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-12-07 19:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Andres Freund <[email protected]>
2022-03-03 15:00 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-03-07 14:45 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-08 15:00 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-03 18:28 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-03-04 11:29 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-03-04 12:20 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-06-06 06:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-23 18:39 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Alvaro Herrera <[email protected]>
2022-02-23 19:22 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Justin Pryzby <[email protected]>
2022-02-24 17:14 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-24 18:53 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-25 07:03 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-25 15:23 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-25 16:35 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Julien Rouhaud <[email protected]>
2022-02-25 14:56 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-17 11:11 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Matthias van de Meent <[email protected]>
2022-02-18 06:32 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2022-02-22 14:40 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Ashutosh Sharma <[email protected]>
2022-02-23 13:37 ` Re: Report checkpoint progress with pg_stat_progress_checkpoint (was: Report checkpoint progress in server logs) Nitin Jadhav <[email protected]>
2024-12-30 12:44 [PATCH v26 8/9] Row pattern recognition patch (typedefs.list). Tatsuo Ishii <[email protected]>
2025-12-13 17:13 [PATCH 1/3] stress test for repack concurrently Mikhail Nikalayeu <[email protected]>
2025-12-13 17:13 [PATCH 1/3] stress test for repack concurrently Mikhail Nikalayeu <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox