From: Bertrand Drouvot Date: Wed, 14 Aug 2024 08:46:05 +0000 Subject: [PATCH v1] Add contrib/pg_logicalsnapinspect Provides SQL functions that allow to inspect the contents of serialized logical snapshots of a running database cluster, which is useful for debugging or educational purposes. --- contrib/Makefile | 1 + contrib/meson.build | 1 + contrib/pg_logicalsnapinspect/.gitignore | 4 + contrib/pg_logicalsnapinspect/Makefile | 31 +++ .../expected/logical_snapshot_inspect.out | 52 ++++ .../logicalsnapinspect.conf | 1 + contrib/pg_logicalsnapinspect/meson.build | 39 +++ .../pg_logicalsnapinspect--1.0.sql | 43 +++ .../pg_logicalsnapinspect.c | 249 ++++++++++++++++++ .../pg_logicalsnapinspect.control | 5 + .../specs/logical_snapshot_inspect.spec | 34 +++ doc/src/sgml/contrib.sgml | 1 + doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/pglogicalsnapinspect.sgml | 144 ++++++++++ src/backend/replication/logical/snapbuild.c | 189 +------------ src/include/port/pg_crc32c.h | 16 +- src/include/replication/snapbuild.h | 186 ++++++++++++- 17 files changed, 800 insertions(+), 197 deletions(-) 7.7% contrib/pg_logicalsnapinspect/expected/ 5.8% contrib/pg_logicalsnapinspect/specs/ 33.2% contrib/pg_logicalsnapinspect/ 13.4% doc/src/sgml/ 17.5% src/backend/replication/logical/ 4.2% src/include/port/ 17.7% src/include/replication/ diff --git a/contrib/Makefile b/contrib/Makefile index abd780f277..a379ce30c8 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -32,6 +32,7 @@ SUBDIRS = \ passwordcheck \ pg_buffercache \ pg_freespacemap \ + pg_logicalsnapinspect \ pg_prewarm \ pg_stat_statements \ pg_surgery \ diff --git a/contrib/meson.build b/contrib/meson.build index 14a8906865..d54009bfe5 100644 --- a/contrib/meson.build +++ b/contrib/meson.build @@ -46,6 +46,7 @@ subdir('passwordcheck') subdir('pg_buffercache') subdir('pgcrypto') subdir('pg_freespacemap') +subdir('pg_logicalsnapinspect') subdir('pg_prewarm') subdir('pgrowlocks') subdir('pg_stat_statements') diff --git a/contrib/pg_logicalsnapinspect/.gitignore b/contrib/pg_logicalsnapinspect/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/contrib/pg_logicalsnapinspect/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/contrib/pg_logicalsnapinspect/Makefile b/contrib/pg_logicalsnapinspect/Makefile new file mode 100644 index 0000000000..aef1d9aa87 --- /dev/null +++ b/contrib/pg_logicalsnapinspect/Makefile @@ -0,0 +1,31 @@ +# contrib/pg_logicalsnapinspect/Makefile + +MODULE_big = pg_logicalsnapinspect +OBJS = \ + $(WIN32RES) \ + pg_logicalsnapinspect.o +PGFILEDESC = "pg_logicalsnapinspect - functions to inspect logical snapshots" + +EXTENSION = pg_logicalsnapinspect +DATA = pg_logicalsnapinspect--1.0.sql + +EXTRA_INSTALL = contrib/test_decoding + +ISOLATION = logical_snapshot_inspect + +ISOLATION_OPTS = --temp-config $(top_srcdir)/contrib/pg_logicalsnapinspect/logicalsnapinspect.conf + +# Disabled because these tests require "wal_level=logical", which +# some installcheck users do not have (e.g. buildfarm clients). +NO_INSTALLCHECK = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/pg_logicalsnapinspect +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/pg_logicalsnapinspect/expected/logical_snapshot_inspect.out b/contrib/pg_logicalsnapinspect/expected/logical_snapshot_inspect.out new file mode 100644 index 0000000000..749cd4642d --- /dev/null +++ b/contrib/pg_logicalsnapinspect/expected/logical_snapshot_inspect.out @@ -0,0 +1,52 @@ +Parsed test spec with 2 sessions + +starting permutation: s0_init s0_begin s0_savepoint s0_truncate s1_checkpoint s1_get_changes s0_commit s0_begin s0_insert s1_checkpoint s1_get_changes s0_commit s1_get_changes s1_get_logical_snapshot_info s1_get_logical_snapshot_meta +step s0_init: SELECT 'init' FROM pg_create_logical_replication_slot('isolation_slot', 'test_decoding'); +?column? +-------- +init +(1 row) + +step s0_begin: BEGIN; +step s0_savepoint: SAVEPOINT sp1; +step s0_truncate: TRUNCATE tbl1; +step s1_checkpoint: CHECKPOINT; +step s1_get_changes: SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL, NULL, 'skip-empty-xacts', '1', 'include-xids', '0'); +data +---- +(0 rows) + +step s0_commit: COMMIT; +step s0_begin: BEGIN; +step s0_insert: INSERT INTO tbl1 VALUES (1); +step s1_checkpoint: CHECKPOINT; +step s1_get_changes: SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL, NULL, 'skip-empty-xacts', '1', 'include-xids', '0'); +data +--------------------------------------- +BEGIN +table public.tbl1: TRUNCATE: (no-flags) +COMMIT +(3 rows) + +step s0_commit: COMMIT; +step s1_get_changes: SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL, NULL, 'skip-empty-xacts', '1', 'include-xids', '0'); +data +------------------------------------------------------------- +BEGIN +table public.tbl1: INSERT: val1[integer]:1 val2[integer]:null +COMMIT +(3 rows) + +step s1_get_logical_snapshot_info: SELECT (pg_get_logical_snapshot_info(f.name::pg_lsn)).state,(pg_get_logical_snapshot_info(f.name::pg_lsn)).catchange_count,array_length((pg_get_logical_snapshot_info(f.name::pg_lsn)).catchange_xip,1),(pg_get_logical_snapshot_info(f.name::pg_lsn)).committed_count,array_length((pg_get_logical_snapshot_info(f.name::pg_lsn)).committed_xip,1) FROM (SELECT replace(replace(name,'.snap',''),'-','/') AS name FROM pg_ls_logicalsnapdir()) AS f ORDER BY 2; +state|catchange_count|array_length|committed_count|array_length +-----+---------------+------------+---------------+------------ + 2| 0| | 2| 2 + 2| 2| 2| 0| +(2 rows) + +step s1_get_logical_snapshot_meta: SELECT COUNT((pg_get_logical_snapshot_meta(f.name::pg_lsn))) FROM (SELECT replace(replace(name,'.snap',''),'-','/') AS name FROM pg_ls_logicalsnapdir()) AS f; +count +----- + 2 +(1 row) + diff --git a/contrib/pg_logicalsnapinspect/logicalsnapinspect.conf b/contrib/pg_logicalsnapinspect/logicalsnapinspect.conf new file mode 100644 index 0000000000..e3d257315f --- /dev/null +++ b/contrib/pg_logicalsnapinspect/logicalsnapinspect.conf @@ -0,0 +1 @@ +wal_level = logical diff --git a/contrib/pg_logicalsnapinspect/meson.build b/contrib/pg_logicalsnapinspect/meson.build new file mode 100644 index 0000000000..9f2c2bb45b --- /dev/null +++ b/contrib/pg_logicalsnapinspect/meson.build @@ -0,0 +1,39 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +pg_logicalsnapinspect_sources = files('pg_logicalsnapinspect.c') + +if host_system == 'windows' + pg_logicalsnapinspect_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'pg_logicalsnapinspect', + '--FILEDESC', 'pg_logicalsnapinspect - functions to inspect contents of logical snapshots',]) +endif + +pg_logicalsnapinspect = shared_module('pg_logicalsnapinspect', + pg_logicalsnapinspect_sources, + kwargs: contrib_mod_args + { + 'dependencies': contrib_mod_args['dependencies'], + }, +) +contrib_targets += pg_logicalsnapinspect + +install_data( + 'pg_logicalsnapinspect.control', + 'pg_logicalsnapinspect--1.0.sql', + kwargs: contrib_data_args, +) + +tests += { + 'name': 'pg_logicalsnapinspect', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'isolation': { + 'specs': [ + 'logical_snapshot_inspect', + ], + 'regress_args': [ + '--temp-config', files('logicalsnapinspect.conf'), + ], + # see above + 'runningcheck': false, + }, +} diff --git a/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect--1.0.sql b/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect--1.0.sql new file mode 100644 index 0000000000..0fcc8aa816 --- /dev/null +++ b/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect--1.0.sql @@ -0,0 +1,43 @@ +/* contrib/pg_logicalsnapinspect/pg_logicalsnapinspect--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION pg_logicalsnapinspect" to load this file. \quit + +-- +-- pg_get_logical_snapshot_meta() +-- +CREATE FUNCTION pg_get_logical_snapshot_meta(IN in_lsn pg_lsn, + OUT magic int4, + OUT checksum int4, + OUT version int4 +) +AS 'MODULE_PATHNAME', 'pg_get_logical_snapshot_meta' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_logical_snapshot_meta(pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_logical_snapshot_meta(pg_lsn) TO pg_read_server_files; + +-- +-- pg_get_logical_snapshot_info() +-- +CREATE FUNCTION pg_get_logical_snapshot_info(IN in_lsn pg_lsn, + OUT state int2, + OUT xmin xid, + OUT xmax xid, + OUT start_decoding_at pg_lsn, + OUT two_phase_at pg_lsn, + OUT initial_xmin_horizon xid, + OUT building_full_snapshot boolean, + OUT in_slot_creation boolean, + OUT last_serialized_snapshot pg_lsn, + OUT next_phase_at xid, + OUT committed_count int8, + OUT committed_xip xid[], + OUT catchange_count int8, + OUT catchange_xip xid[] +) +AS 'MODULE_PATHNAME', 'pg_get_logical_snapshot_info' +LANGUAGE C STRICT PARALLEL SAFE; + +REVOKE EXECUTE ON FUNCTION pg_get_logical_snapshot_info(pg_lsn) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_get_logical_snapshot_info(pg_lsn) TO pg_read_server_files; diff --git a/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect.c b/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect.c new file mode 100644 index 0000000000..874129d01f --- /dev/null +++ b/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect.c @@ -0,0 +1,249 @@ +/*------------------------------------------------------------------------- + * + * pg_logicalsnapinspect.c + * Functions to inspect contents of PostgreSQL logical snapshots + * + * Copyright (c) 2024, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/pg_logicalsnapinspect/pg_logicalsnapinspect.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "funcapi.h" +#include "port/pg_crc32c.h" +#include "replication/snapbuild.h" +#include "utils/array.h" +#include "utils/pg_lsn.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(pg_get_logical_snapshot_meta); +PG_FUNCTION_INFO_V1(pg_get_logical_snapshot_info); + +static void ValidateSnapshotFile(XLogRecPtr lsn, SnapBuildOnDisk *ondisk, + const char *path); + +/* + * NOTE: For any code change or issue fix here, it is highly recommended to + * give a thought about doing the same in SnapBuildRestore() as well. + */ + +/* + * Validate the logical snapshot file. + */ +static void +ValidateSnapshotFile(XLogRecPtr lsn, SnapBuildOnDisk *ondisk, const char *path) +{ + int fd; + Size sz; + pg_crc32c checksum; + MemoryContext context; + + context = AllocSetContextCreate(CurrentMemoryContext, + "logicalsnapshot inspect context", + ALLOCSET_DEFAULT_SIZES); + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + + if (fd < 0 && errno == ENOENT) + ereport(ERROR, + errmsg("file \"%s\" does not exist", path)); + else if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + + /* ---- + * Make sure the snapshot had been stored safely to disk, that's normally + * cheap. + * Note that we do not need PANIC here, nobody will be able to use the + * slot without fsyncing, and saving it won't succeed without an fsync() + * either... + * ---- + */ + fsync_fname(path, false); + fsync_fname("pg_logical/snapshots", true); + + + /* read statically sized portion of snapshot */ + SnapBuildRestoreContents(fd, (char *) ondisk, SnapBuildOnDiskConstantSize, path); + + if (ondisk->magic != SNAPBUILD_MAGIC) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("snapbuild state file \"%s\" has wrong magic number: %u instead of %u", + path, ondisk->magic, SNAPBUILD_MAGIC))); + + if (ondisk->version != SNAPBUILD_VERSION) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("snapbuild state file \"%s\" has unsupported version: %u instead of %u", + path, ondisk->version, SNAPBUILD_VERSION))); + + INIT_CRC32C(checksum); + COMP_CRC32C(checksum, + ((char *) ondisk) + SnapBuildOnDiskNotChecksummedSize, + SnapBuildOnDiskConstantSize - SnapBuildOnDiskNotChecksummedSize); + + /* read SnapBuild */ + SnapBuildRestoreContents(fd, (char *) &ondisk->builder, sizeof(SnapBuild), path); + COMP_CRC32C(checksum, &ondisk->builder, sizeof(SnapBuild)); + + ondisk->builder.context = context; + + /* restore committed xacts information */ + if (ondisk->builder.committed.xcnt > 0) + { + sz = sizeof(TransactionId) * ondisk->builder.committed.xcnt; + ondisk->builder.committed.xip = MemoryContextAllocZero(ondisk->builder.context, sz); + SnapBuildRestoreContents(fd, (char *) ondisk->builder.committed.xip, sz, path); + COMP_CRC32C(checksum, ondisk->builder.committed.xip, sz); + } + + /* restore catalog modifying xacts information */ + if (ondisk->builder.catchange.xcnt > 0) + { + sz = sizeof(TransactionId) * ondisk->builder.catchange.xcnt; + ondisk->builder.catchange.xip = MemoryContextAllocZero(ondisk->builder.context, sz); + SnapBuildRestoreContents(fd, (char *) ondisk->builder.catchange.xip, sz, path); + COMP_CRC32C(checksum, ondisk->builder.catchange.xip, sz); + } + + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + FIN_CRC32C(checksum); + + /* verify checksum of what we've read */ + if (!EQ_CRC32C(checksum, ondisk->checksum)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("checksum mismatch for snapbuild state file \"%s\": is %u, should be %u", + path, checksum, ondisk->checksum))); +} + +/* + * Retrieve the logical snapshot file metadata. + */ +Datum +pg_get_logical_snapshot_meta(PG_FUNCTION_ARGS) +{ +#define PG_GET_LOGICAL_SNAPSHOT_META_COLS 3 + SnapBuildOnDisk ondisk; + XLogRecPtr lsn; + HeapTuple tuple; + Datum values[PG_GET_LOGICAL_SNAPSHOT_META_COLS]; + bool nulls[PG_GET_LOGICAL_SNAPSHOT_META_COLS]; + TupleDesc tupdesc; + char path[MAXPGPATH]; + + lsn = PG_GETARG_LSN(0); + + sprintf(path, "pg_logical/snapshots/%X-%X.snap", + LSN_FORMAT_ARGS(lsn)); + + ValidateSnapshotFile(lsn, &ondisk, path); + + /* Build a tuple descriptor for our result type. */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + memset(nulls, 0, sizeof(nulls)); + + values[0] = Int32GetDatum(ondisk.magic); + values[1] = Int32GetDatum(ondisk.checksum); + values[2] = Int32GetDatum(ondisk.version); + + tuple = heap_form_tuple(tupdesc, values, nulls); + + MemoryContextReset(ondisk.builder.context); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); + +#undef PG_GET_LOGICAL_SNAPSHOT_META_COLS +} + +Datum +pg_get_logical_snapshot_info(PG_FUNCTION_ARGS) +{ +#define PG_GET_LOGICAL_SNAPSHOT_INFO_COLS 14 + SnapBuildOnDisk ondisk; + XLogRecPtr lsn; + HeapTuple tuple; + Datum values[PG_GET_LOGICAL_SNAPSHOT_INFO_COLS]; + bool nulls[PG_GET_LOGICAL_SNAPSHOT_INFO_COLS]; + TupleDesc tupdesc; + char path[MAXPGPATH]; + + lsn = PG_GETARG_LSN(0); + + sprintf(path, "pg_logical/snapshots/%X-%X.snap", + LSN_FORMAT_ARGS(lsn)); + + ValidateSnapshotFile(lsn, &ondisk, path); + + /* Build a tuple descriptor for our result type. */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + memset(nulls, 0, sizeof(nulls)); + + values[0] = Int16GetDatum(ondisk.builder.state); + values[1] = TransactionIdGetDatum(ondisk.builder.xmin); + values[2] = TransactionIdGetDatum(ondisk.builder.xmax); + values[3] = LSNGetDatum(ondisk.builder.start_decoding_at); + values[4] = LSNGetDatum(ondisk.builder.two_phase_at); + values[5] = TransactionIdGetDatum(ondisk.builder.initial_xmin_horizon); + values[6] = BoolGetDatum(ondisk.builder.building_full_snapshot); + values[7] = BoolGetDatum(ondisk.builder.in_slot_creation); + values[8] = LSNGetDatum(ondisk.builder.last_serialized_snapshot); + values[9] = TransactionIdGetDatum(ondisk.builder.next_phase_at); + values[10] = Int64GetDatum(ondisk.builder.committed.xcnt); + + if (ondisk.builder.committed.xcnt > 0) + { + Datum *arrayelems; + int narrayelems; + + arrayelems = (Datum *) palloc(ondisk.builder.committed.xcnt * sizeof(Datum)); + narrayelems = 0; + + for (narrayelems = 0; narrayelems < ondisk.builder.committed.xcnt; narrayelems++) + arrayelems[narrayelems] = Int64GetDatum((int64) ondisk.builder.committed.xip[narrayelems]); + + values[11] = PointerGetDatum(construct_array_builtin(arrayelems, narrayelems, INT8OID)); + } + else + nulls[11] = true; + + values[12] = Int64GetDatum(ondisk.builder.catchange.xcnt); + + if (ondisk.builder.catchange.xcnt > 0) + { + Datum *arrayelems; + int narrayelems; + + arrayelems = (Datum *) palloc(ondisk.builder.catchange.xcnt * sizeof(Datum)); + narrayelems = 0; + + for (narrayelems = 0; narrayelems < ondisk.builder.catchange.xcnt; narrayelems++) + arrayelems[narrayelems] = Int64GetDatum((int64) ondisk.builder.catchange.xip[narrayelems]); + + values[13] = PointerGetDatum(construct_array_builtin(arrayelems, narrayelems, INT8OID)); + } + else + nulls[13] = true; + + tuple = heap_form_tuple(tupdesc, values, nulls); + + MemoryContextReset(ondisk.builder.context); + + PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); + +#undef PG_GET_LOGICAL_SNAPSHOT_INFO_COLS +} diff --git a/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect.control b/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect.control new file mode 100644 index 0000000000..b366ccb10c --- /dev/null +++ b/contrib/pg_logicalsnapinspect/pg_logicalsnapinspect.control @@ -0,0 +1,5 @@ +# pg_logicalsnapinspect extension +comment = 'functions to inspect contents of logical snapshot' +default_version = '1.0' +module_pathname = '$libdir/pg_logicalsnapinspect' +relocatable = true diff --git a/contrib/pg_logicalsnapinspect/specs/logical_snapshot_inspect.spec b/contrib/pg_logicalsnapinspect/specs/logical_snapshot_inspect.spec new file mode 100644 index 0000000000..6fd2c338ca --- /dev/null +++ b/contrib/pg_logicalsnapinspect/specs/logical_snapshot_inspect.spec @@ -0,0 +1,34 @@ +# Test the pg_logicalsnapinspect functions: that needs some permutation to +# ensure that we are creating multiple logical snapshots and that one of them +# contains ongoing catalogs changes. +setup +{ + DROP TABLE IF EXISTS tbl1; + CREATE TABLE tbl1 (val1 integer, val2 integer); + CREATE EXTENSION pg_logicalsnapinspect; +} + +teardown +{ + DROP TABLE tbl1; + SELECT 'stop' FROM pg_drop_replication_slot('isolation_slot'); + DROP EXTENSION pg_logicalsnapinspect; +} + +session "s0" +setup { SET synchronous_commit=on; } +step "s0_init" { SELECT 'init' FROM pg_create_logical_replication_slot('isolation_slot', 'test_decoding'); } +step "s0_begin" { BEGIN; } +step "s0_savepoint" { SAVEPOINT sp1; } +step "s0_truncate" { TRUNCATE tbl1; } +step "s0_insert" { INSERT INTO tbl1 VALUES (1); } +step "s0_commit" { COMMIT; } + +session "s1" +setup { SET synchronous_commit=on; } +step "s1_checkpoint" { CHECKPOINT; } +step "s1_get_changes" { SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL, NULL, 'skip-empty-xacts', '1', 'include-xids', '0'); } +step "s1_get_logical_snapshot_meta" { SELECT COUNT((pg_get_logical_snapshot_meta(f.name::pg_lsn))) FROM (SELECT replace(replace(name,'.snap',''),'-','/') AS name FROM pg_ls_logicalsnapdir()) AS f; } +step "s1_get_logical_snapshot_info" { SELECT (pg_get_logical_snapshot_info(f.name::pg_lsn)).state,(pg_get_logical_snapshot_info(f.name::pg_lsn)).catchange_count,array_length((pg_get_logical_snapshot_info(f.name::pg_lsn)).catchange_xip,1),(pg_get_logical_snapshot_info(f.name::pg_lsn)).committed_count,array_length((pg_get_logical_snapshot_info(f.name::pg_lsn)).committed_xip,1) FROM (SELECT replace(replace(name,'.snap',''),'-','/') AS name FROM pg_ls_logicalsnapdir()) AS f ORDER BY 2; } + +permutation "s0_init" "s0_begin" "s0_savepoint" "s0_truncate" "s1_checkpoint" "s1_get_changes" "s0_commit" "s0_begin" "s0_insert" "s1_checkpoint" "s1_get_changes" "s0_commit" "s1_get_changes" "s1_get_logical_snapshot_info" "s1_get_logical_snapshot_meta" diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index 44639a8dca..f7b1cd85ee 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -154,6 +154,7 @@ CREATE EXTENSION extension_name; &pgbuffercache; &pgcrypto; &pgfreespacemap; + &pglogicalsnapinspect; &pgprewarm; &pgrowlocks; &pgstatstatements; diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index a7ff5f8264..94b650915d 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -143,6 +143,7 @@ + diff --git a/doc/src/sgml/pglogicalsnapinspect.sgml b/doc/src/sgml/pglogicalsnapinspect.sgml new file mode 100644 index 0000000000..5e005ab124 --- /dev/null +++ b/doc/src/sgml/pglogicalsnapinspect.sgml @@ -0,0 +1,144 @@ + + + + pg_logicalsnapinspect — logical snapshot inspection + + + pg_logicalsnapinspect + + + + The pg_logicalsnapinspect module provides SQL functions + that allow you to inspect the contents of serialized logical snapshots of a + running PostgreSQL database cluster, which is useful + for debugging or educational purposes. + + + + + The pg_logicalsnapinspect functions are called + using an LSN argument that can be extracted from the output name of the + pg_ls_logicalsnapdir() function. + + + + + General Functions + + + + + pg_get_logical_snapshot_meta(in_lsn pg_lsn) returns record + + + + + Gets logical snapshot metadata about a snapshot file that is located in + the pg_logical/snapshots directory. + The in_lsn argument can be extracted from the + snapshot file name. + example: + +postgres=# SELECT * FROM pg_ls_logicalsnapdir(); +-[ RECORD 1 ]+----------------------- +name | 0-40796E18.snap +size | 152 +modification | 2024-08-14 16:36:32+00 + +postgres=# SELECT * FROM pg_get_logical_snapshot_meta('0/40796E18'); +-[ RECORD 1 ]-------- +magic | 1369563137 +checksum | 1028045905 +version | 6 + +postgres=# SELECT (pg_get_logical_snapshot_meta(f.name::pg_lsn)).* + FROM (SELECT replace(replace(name,'.snap',''),'-','/') AS name + FROM pg_ls_logicalsnapdir()) AS f; +-[ RECORD 1 ]-------- +magic | 1369563137 +checksum | 1028045905 +version | 6 + + + + If in_lsn does not match a snapshot file, the + function raises an error. + + + + + + + pg_get_logical_snapshot_info(in_lsn pg_lsn) returns record + + + + + Gets logical snapshot information about a snapshot file that is located in + the pg_logical/snapshots directory. + The in_lsn argument can be extracted from the + snapshot file name. + example: + +postgres=# SELECT * FROM pg_ls_logicalsnapdir(); +-[ RECORD 1 ]+----------------------- +name | 0-40796E18.snap +size | 152 +modification | 2024-08-14 16:36:32+00 + +postgres=# SELECT * FROM pg_get_logical_snapshot_info('0/40796E18'); +-[ RECORD 1 ]------------+----------- +state | 2 +xmin | 751 +xmax | 751 +start_decoding_at | 0/40796AF8 +two_phase_at | 0/40796AF8 +initial_xmin_horizon | 0 +building_full_snapshot | f +in_slot_creation | f +last_serialized_snapshot | 0/0 +next_phase_at | 0 +committed_count | 0 +committed_xip | +catchange_count | 2 +catchange_xip | {751,752} + +postgres=# SELECT (pg_get_logical_snapshot_info(f.name::pg_lsn)).* + FROM (SELECT replace(replace(name,'.snap',''),'-','/') AS name + FROM pg_ls_logicalsnapdir()) AS f; +-[ RECORD 1 ]------------+----------- +state | 2 +xmin | 751 +xmax | 751 +start_decoding_at | 0/40796AF8 +two_phase_at | 0/40796AF8 +initial_xmin_horizon | 0 +building_full_snapshot | f +in_slot_creation | f +last_serialized_snapshot | 0/0 +next_phase_at | 0 +committed_count | 0 +committed_xip | +catchange_count | 2 +catchange_xip | {751,752} + + + + If in_lsn does not match a snapshot file, the + function raises an error. + + + + + + + + + Author + + + Bertrand Drouvot bertranddrouvot.pg@gmail.com + + + + diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index ae676145e6..b9b8e894b6 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -143,146 +143,6 @@ #include "utils/memutils.h" #include "utils/snapmgr.h" #include "utils/snapshot.h" - -/* - * This struct contains the current state of the snapshot building - * machinery. Besides a forward declaration in the header, it is not exposed - * to the public, so we can easily change its contents. - */ -struct SnapBuild -{ - /* how far are we along building our first full snapshot */ - SnapBuildState state; - - /* private memory context used to allocate memory for this module. */ - MemoryContext context; - - /* all transactions < than this have committed/aborted */ - TransactionId xmin; - - /* all transactions >= than this are uncommitted */ - TransactionId xmax; - - /* - * Don't replay commits from an LSN < this LSN. This can be set externally - * but it will also be advanced (never retreat) from within snapbuild.c. - */ - XLogRecPtr start_decoding_at; - - /* - * LSN at which two-phase decoding was enabled or LSN at which we found a - * consistent point at the time of slot creation. - * - * The prepared transactions, that were skipped because previously - * two-phase was not enabled or are not covered by initial snapshot, need - * to be sent later along with commit prepared and they must be before - * this point. - */ - XLogRecPtr two_phase_at; - - /* - * Don't start decoding WAL until the "xl_running_xacts" information - * indicates there are no running xids with an xid smaller than this. - */ - TransactionId initial_xmin_horizon; - - /* Indicates if we are building full snapshot or just catalog one. */ - bool building_full_snapshot; - - /* - * Indicates if we are using the snapshot builder for the creation of a - * logical replication slot. If it's true, the start point for decoding - * changes is not determined yet. So we skip snapshot restores to properly - * find the start point. See SnapBuildFindSnapshot() for details. - */ - bool in_slot_creation; - - /* - * Snapshot that's valid to see the catalog state seen at this moment. - */ - Snapshot snapshot; - - /* - * LSN of the last location we are sure a snapshot has been serialized to. - */ - XLogRecPtr last_serialized_snapshot; - - /* - * The reorderbuffer we need to update with usable snapshots et al. - */ - ReorderBuffer *reorder; - - /* - * TransactionId at which the next phase of initial snapshot building will - * happen. InvalidTransactionId if not known (i.e. SNAPBUILD_START), or - * when no next phase necessary (SNAPBUILD_CONSISTENT). - */ - TransactionId next_phase_at; - - /* - * Array of transactions which could have catalog changes that committed - * between xmin and xmax. - */ - struct - { - /* number of committed transactions */ - size_t xcnt; - - /* available space for committed transactions */ - size_t xcnt_space; - - /* - * Until we reach a CONSISTENT state, we record commits of all - * transactions, not just the catalog changing ones. Record when that - * changes so we know we cannot export a snapshot safely anymore. - */ - bool includes_all_transactions; - - /* - * Array of committed transactions that have modified the catalog. - * - * As this array is frequently modified we do *not* keep it in - * xidComparator order. Instead we sort the array when building & - * distributing a snapshot. - * - * TODO: It's unclear whether that reasoning has much merit. Every - * time we add something here after becoming consistent will also - * require distributing a snapshot. Storing them sorted would - * potentially also make it easier to purge (but more complicated wrt - * wraparound?). Should be improved if sorting while building the - * snapshot shows up in profiles. - */ - TransactionId *xip; - } committed; - - /* - * Array of transactions and subtransactions that had modified catalogs - * and were running when the snapshot was serialized. - * - * We normally rely on some WAL record types such as HEAP2_NEW_CID to know - * if the transaction has changed the catalog. But it could happen that - * the logical decoding decodes only the commit record of the transaction - * after restoring the previously serialized snapshot in which case we - * will miss adding the xid to the snapshot and end up looking at the - * catalogs with the wrong snapshot. - * - * Now to avoid the above problem, we serialize the transactions that had - * modified the catalogs and are still running at the time of snapshot - * serialization. We fill this array while restoring the snapshot and then - * refer it while decoding commit to ensure if the xact has modified the - * catalog. We discard this array when all the xids in the list become old - * enough to matter. See SnapBuildPurgeOlderTxn for details. - */ - struct - { - /* number of transactions */ - size_t xcnt; - - /* This array must be sorted in xidComparator order */ - TransactionId *xip; - } catchange; -}; - /* * Starting a transaction -- which we need to do while exporting a snapshot -- * removes knowledge about the previously used resowner, so we save it here. @@ -312,7 +172,6 @@ static void SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutof /* serialization functions */ static void SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn); static bool SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn); -static void SnapBuildRestoreContents(int fd, char *dest, Size size, const char *path); /* * Allocate a new snapshot builder. @@ -1557,48 +1416,6 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff) } } -/* ----------------------------------- - * Snapshot serialization support - * ----------------------------------- - */ - -/* - * We store current state of struct SnapBuild on disk in the following manner: - * - * struct SnapBuildOnDisk; - * TransactionId * committed.xcnt; (*not xcnt_space*) - * TransactionId * catchange.xcnt; - * - */ -typedef struct SnapBuildOnDisk -{ - /* first part of this struct needs to be version independent */ - - /* data not covered by checksum */ - uint32 magic; - pg_crc32c checksum; - - /* data covered by checksum */ - - /* version, in case we want to support pg_upgrade */ - uint32 version; - /* how large is the on disk data, excluding the constant sized part */ - uint32 length; - - /* version dependent part */ - SnapBuild builder; - - /* variable amount of TransactionIds follows */ -} SnapBuildOnDisk; - -#define SnapBuildOnDiskConstantSize \ - offsetof(SnapBuildOnDisk, builder) -#define SnapBuildOnDiskNotChecksummedSize \ - offsetof(SnapBuildOnDisk, version) - -#define SNAPBUILD_MAGIC 0x51A1E001 -#define SNAPBUILD_VERSION 6 - /* * Store/Load a snapshot from disk, depending on the snapshot builder's state. * @@ -1857,6 +1674,10 @@ out: /* * Restore a snapshot into 'builder' if previously one has been stored at the * location indicated by 'lsn'. Returns true if successful, false otherwise. + * + * NOTE: For any code change or issue fix here, it is highly recommended to + * give a thought about doing the same in pg_logicalsnapinspect contrib module + * as well. */ static bool SnapBuildRestore(SnapBuild *builder, XLogRecPtr lsn) @@ -2030,7 +1851,7 @@ snapshot_not_interesting: /* * Read the contents of the serialized snapshot to 'dest'. */ -static void +void SnapBuildRestoreContents(int fd, char *dest, Size size, const char *path) { int readBytes; diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h index 63c8e3a00b..cfc8c07944 100644 --- a/src/include/port/pg_crc32c.h +++ b/src/include/port/pg_crc32c.h @@ -47,7 +47,7 @@ typedef uint32 pg_crc32c; ((crc) = pg_comp_crc32c_sse42((crc), (data), (len))) #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) -extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); #elif defined(USE_ARMV8_CRC32C) /* Use ARMv8 CRC Extension instructions. */ @@ -56,7 +56,7 @@ extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t le ((crc) = pg_comp_crc32c_armv8((crc), (data), (len))) #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) -extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); #elif defined(USE_LOONGARCH_CRC32C) /* Use LoongArch CRCC instructions. */ @@ -65,7 +65,7 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le ((crc) = pg_comp_crc32c_loongarch((crc), (data), (len))) #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) -extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_t len); #elif defined(USE_SSE42_CRC32C_WITH_RUNTIME_CHECK) || defined(USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK) @@ -77,14 +77,14 @@ extern pg_crc32c pg_comp_crc32c_loongarch(pg_crc32c crc, const void *data, size_ ((crc) = pg_comp_crc32c((crc), (data), (len))) #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) -extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); -extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK -extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); #endif #ifdef USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK -extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t len); #endif #else @@ -103,7 +103,7 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) #endif -extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +extern PGDLLIMPORT pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); #endif diff --git a/src/include/replication/snapbuild.h b/src/include/replication/snapbuild.h index caa5113ff8..a4617c5197 100644 --- a/src/include/replication/snapbuild.h +++ b/src/include/replication/snapbuild.h @@ -13,8 +13,22 @@ #define SNAPBUILD_H #include "access/xlogdefs.h" +#include "replication/reorderbuffer.h" #include "utils/snapmgr.h" +/* ----------------------------------- + * Snapshot serialization support + * ----------------------------------- + */ + +#define SnapBuildOnDiskConstantSize \ + offsetof(SnapBuildOnDisk, builder) +#define SnapBuildOnDiskNotChecksummedSize \ + offsetof(SnapBuildOnDisk, version) + +#define SNAPBUILD_MAGIC 0x51A1E001 +#define SNAPBUILD_VERSION 6 + typedef enum { /* @@ -46,12 +60,173 @@ typedef enum SNAPBUILD_CONSISTENT = 2, } SnapBuildState; -/* forward declare so we don't have to expose the struct to the public */ -struct SnapBuild; -typedef struct SnapBuild SnapBuild; +/* + * This struct contains the current state of the snapshot building + * machinery. It is exposed to the public, so pay attention when changing its + * contents. + */ +typedef struct SnapBuild +{ + /* how far are we along building our first full snapshot */ + SnapBuildState state; + + /* private memory context used to allocate memory for this module. */ + MemoryContext context; + + /* all transactions < than this have committed/aborted */ + TransactionId xmin; + + /* all transactions >= than this are uncommitted */ + TransactionId xmax; + + /* + * Don't replay commits from an LSN < this LSN. This can be set externally + * but it will also be advanced (never retreat) from within snapbuild.c. + */ + XLogRecPtr start_decoding_at; + + /* + * LSN at which two-phase decoding was enabled or LSN at which we found a + * consistent point at the time of slot creation. + * + * The prepared transactions, that were skipped because previously + * two-phase was not enabled or are not covered by initial snapshot, need + * to be sent later along with commit prepared and they must be before + * this point. + */ + XLogRecPtr two_phase_at; + + /* + * Don't start decoding WAL until the "xl_running_xacts" information + * indicates there are no running xids with an xid smaller than this. + */ + TransactionId initial_xmin_horizon; + + /* Indicates if we are building full snapshot or just catalog one. */ + bool building_full_snapshot; + + /* + * Indicates if we are using the snapshot builder for the creation of a + * logical replication slot. If it's true, the start point for decoding + * changes is not determined yet. So we skip snapshot restores to properly + * find the start point. See SnapBuildFindSnapshot() for details. + */ + bool in_slot_creation; + + /* + * Snapshot that's valid to see the catalog state seen at this moment. + */ + Snapshot snapshot; + + /* + * LSN of the last location we are sure a snapshot has been serialized to. + */ + XLogRecPtr last_serialized_snapshot; + + /* + * The reorderbuffer we need to update with usable snapshots et al. + */ + ReorderBuffer *reorder; + + /* + * TransactionId at which the next phase of initial snapshot building will + * happen. InvalidTransactionId if not known (i.e. SNAPBUILD_START), or + * when no next phase necessary (SNAPBUILD_CONSISTENT). + */ + TransactionId next_phase_at; + + /* + * Array of transactions which could have catalog changes that committed + * between xmin and xmax. + */ + struct + { + /* number of committed transactions */ + size_t xcnt; + + /* available space for committed transactions */ + size_t xcnt_space; + + /* + * Until we reach a CONSISTENT state, we record commits of all + * transactions, not just the catalog changing ones. Record when that + * changes so we know we cannot export a snapshot safely anymore. + */ + bool includes_all_transactions; + + /* + * Array of committed transactions that have modified the catalog. + * + * As this array is frequently modified we do *not* keep it in + * xidComparator order. Instead we sort the array when building & + * distributing a snapshot. + * + * TODO: It's unclear whether that reasoning has much merit. Every + * time we add something here after becoming consistent will also + * require distributing a snapshot. Storing them sorted would + * potentially also make it easier to purge (but more complicated wrt + * wraparound?). Should be improved if sorting while building the + * snapshot shows up in profiles. + */ + TransactionId *xip; + } committed; + + /* + * Array of transactions and subtransactions that had modified catalogs + * and were running when the snapshot was serialized. + * + * We normally rely on some WAL record types such as HEAP2_NEW_CID to know + * if the transaction has changed the catalog. But it could happen that + * the logical decoding decodes only the commit record of the transaction + * after restoring the previously serialized snapshot in which case we + * will miss adding the xid to the snapshot and end up looking at the + * catalogs with the wrong snapshot. + * + * Now to avoid the above problem, we serialize the transactions that had + * modified the catalogs and are still running at the time of snapshot + * serialization. We fill this array while restoring the snapshot and then + * refer it while decoding commit to ensure if the xact has modified the + * catalog. We discard this array when all the xids in the list become old + * enough to matter. See SnapBuildPurgeOlderTxn for details. + */ + struct + { + /* number of transactions */ + size_t xcnt; + + /* This array must be sorted in xidComparator order */ + TransactionId *xip; + } catchange; +} SnapBuild; + +/* + * We store current state of struct SnapBuild on disk in the following manner: + * + * struct SnapBuildOnDisk; + * TransactionId * committed.xcnt; (*not xcnt_space*) + * TransactionId * catchange.xcnt; + * + */ +typedef struct SnapBuildOnDisk +{ + /* first part of this struct needs to be version independent */ + + /* data not covered by checksum */ + uint32 magic; + pg_crc32c checksum; + + /* data covered by checksum */ + + /* version, in case we want to support pg_upgrade */ + uint32 version; + /* how large is the on disk data, excluding the constant sized part */ + uint32 length; + + /* version dependent part */ + SnapBuild builder; -/* forward declare so we don't have to include reorderbuffer.h */ -struct ReorderBuffer; + /* variable amount of TransactionIds follows */ +} SnapBuildOnDisk; /* forward declare so we don't have to include heapam_xlog.h */ struct xl_heap_new_cid; @@ -94,4 +269,5 @@ extern void SnapBuildSerializationPoint(SnapBuild *builder, XLogRecPtr lsn); extern bool SnapBuildSnapshotExists(XLogRecPtr lsn); +extern void SnapBuildRestoreContents(int fd, char *dest, Size size, const char *path); #endif /* SNAPBUILD_H */ -- 2.34.1 --gdFCECMgkA3+uUp6--