agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH 3/6] Default to zstd.. 3+ messages / 2 participants [nested] [flat]
* [PATCH 3/6] Default to zstd.. @ 2021-03-12 21:35 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 3+ messages in thread From: Justin Pryzby @ 2021-03-12 21:35 UTC (permalink / raw) for CI, not for merge --- .cirrus.yml | 14 ++++++++++++++ configure | 6 ++++-- configure.ac | 2 +- src/backend/access/transam/xlog.c | 2 +- src/backend/utils/misc/guc.c | 2 +- src/tools/ci/windows_build_config.pl | 1 + 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index d10b0a82f9..4735c67608 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -77,6 +77,7 @@ task: mkdir -m 770 /tmp/cores chown root:postgres /tmp/cores sysctl kern.corefile='/tmp/cores/%N.%P.core' + pkg install -y zstd # NB: Intentionally build without --with-llvm. The freebsd image size is # already large enough to make VM startup slow, and even without llvm @@ -184,6 +185,8 @@ task: mkdir -m 770 /tmp/cores chown root:postgres /tmp/cores sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core' + apt-get update + apt-get -y install libzstd-dev configure_script: | su postgres <<-EOF @@ -265,6 +268,7 @@ task: openldap \ openssl \ python \ + zstd \ tcl-tk brew cleanup -s # to reduce cache size @@ -385,6 +389,12 @@ task: set configure_script: + # XXX: not working + - choco install -y cmake + - curl -L -o zstd.zip https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.zip + - unzip zstd.zip + - cd zstd-1.5.2 && vcvarsall x64 && build\VS_scripts\build.VS2017.cmd && cd .. + #- cp zstd-1.5.2/build/VS_scripts/build.VS2017.cmd # copy errors out when using forward slashes - copy src\tools\ci\windows_build_config.pl src\tools\msvc\config.pl - vcvarsall x64 @@ -467,6 +477,10 @@ task: image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest cpu: $CPUS + setup_os_script: | + apt-get update + apt-get -y install libzstd-dev + sysinfo_script: | id uname -a diff --git a/configure b/configure index f3cb5c2b51..b88917fcb5 100755 --- a/configure +++ b/configure @@ -1584,7 +1584,7 @@ Optional Packages: use system time zone data in DIR --without-zlib do not use Zlib --with-lz4 build with LZ4 support - --with-zstd build with ZSTD support + --without-zstd build without Zstd support --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -9070,7 +9070,9 @@ $as_echo "#define USE_ZSTD 1" >>confdefs.h esac else - with_zstd=no + with_zstd=yes + +$as_echo "#define USE_ZSTD 1" >>confdefs.h fi diff --git a/configure.ac b/configure.ac index 19d1a80367..4ae681d8bf 100644 --- a/configure.ac +++ b/configure.ac @@ -1060,7 +1060,7 @@ fi # ZSTD # AC_MSG_CHECKING([whether to build with ZSTD support]) -PGAC_ARG_BOOL(with, zstd, no, [build with ZSTD support], +PGAC_ARG_BOOL(with, zstd, yes, [build with ZSTD support], [AC_DEFINE([USE_ZSTD], 1, [Define to 1 to build with ZSTD support. (--with-zstd)])]) AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 0d2bd7a357..9aa4d3eaea 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -120,7 +120,7 @@ char *XLogArchiveCommand = NULL; bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; -int wal_compression = WAL_COMPRESSION_NONE; +int wal_compression = WAL_COMPRESSION_ZSTD; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 66e6d664b0..96854b6a2b 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4900,7 +4900,7 @@ static struct config_enum ConfigureNamesEnum[] = NULL }, &wal_compression, - WAL_COMPRESSION_NONE, wal_compression_options, + WAL_COMPRESSION_ZSTD, wal_compression_options, NULL, NULL, NULL }, diff --git a/src/tools/ci/windows_build_config.pl b/src/tools/ci/windows_build_config.pl index b0d4360c74..97ce79ea7f 100644 --- a/src/tools/ci/windows_build_config.pl +++ b/src/tools/ci/windows_build_config.pl @@ -9,5 +9,6 @@ $config->{"asserts"} = 1; $config->{"openssl"} = "c:/openssl/1.1/"; $config->{"perl"} = "c:/strawberry/$ENV{DEFAULT_PERL_VERSION}/perl/"; $config->{"python"} = "c:/python/"; +$config->{"zstd"} = "c:/zstd/"; 1; -- 2.17.1 --pAwQNkOnpTn9IO2O Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0004-Use-GUC-hooks-to-support-compression-level.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH 08/10] Default to zstd.. @ 2021-03-12 21:35 Justin Pryzby <pryzbyj@telsasoft.com> 0 siblings, 0 replies; 3+ messages in thread From: Justin Pryzby @ 2021-03-12 21:35 UTC (permalink / raw) for CI, not for merge --- configure | 6 ++++-- configure.ac | 2 +- src/backend/access/transam/xlog.c | 2 +- src/backend/utils/misc/guc.c | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/configure b/configure index 81e23418b2..253f028fc4 100755 --- a/configure +++ b/configure @@ -1582,7 +1582,7 @@ Optional Packages: use system time zone data in DIR --without-zlib do not use Zlib --without-lz4 build without LZ4 support - --with-zstd build with Zstd compression library + --without-zstd build without Zstd compression library --with-gnu-ld assume the C compiler uses GNU ld [default=no] --with-ssl=LIB use LIB for SSL/TLS support (openssl) --with-openssl obsolete spelling of --with-ssl=openssl @@ -8740,7 +8740,9 @@ $as_echo "#define USE_ZSTD 1" >>confdefs.h esac else - with_zstd=no + with_zstd=yes + +$as_echo "#define USE_ZSTD 1" >>confdefs.h fi diff --git a/configure.ac b/configure.ac index d6f6349067..8d72710fa7 100644 --- a/configure.ac +++ b/configure.ac @@ -1005,7 +1005,7 @@ fi # ZSTD # AC_MSG_CHECKING([whether to build with zstd support]) -PGAC_ARG_BOOL(with, zstd, no, [build with Zstd compression library], +PGAC_ARG_BOOL(with, zstd, yes, [build without Zstd compression library], [AC_DEFINE([USE_ZSTD], 1, [Define to 1 to build with zstd support. (--with-zstd)])]) AC_MSG_RESULT([$with_zstd]) AC_SUBST(with_zstd) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 307eee6626..92023de9f5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -99,7 +99,7 @@ bool EnableHotStandby = false; bool fullPageWrites = true; bool wal_log_hints = false; bool wal_compression = false; -int wal_compression_method = WAL_COMPRESSION_LZ4; +int wal_compression_method = WAL_COMPRESSION_ZSTD; char *wal_consistency_checking_string = NULL; bool *wal_consistency_checking = NULL; bool wal_init_zero = true; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 52f9cd0242..8031e027aa 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4728,7 +4728,7 @@ static struct config_enum ConfigureNamesEnum[] = NULL }, &wal_compression_method, - WAL_COMPRESSION_LZ4, wal_compression_options, + WAL_COMPRESSION_ZSTD, wal_compression_options, NULL, NULL, NULL }, -- 2.17.0 --jozmn01XJZjDjM3N Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0009-Add-zstd-compression-levels.patch" ^ permalink raw reply [nested|flat] 3+ messages in thread
* [PATCH v1] Add per-backend AIO statistics @ 2026-06-11 09:45 Bertrand Drouvot <bertranddrouvot.pg@gmail.com> 0 siblings, 0 replies; 3+ messages in thread From: Bertrand Drouvot @ 2026-06-11 09:45 UTC (permalink / raw) This commit adds per-backend AIO statistics, providing per-backend AIO behavior details. This data can be retrieved with a new system function called pg_stat_get_backend_aio(), that returns the following counters based on the PID provided in input: - started: total number of AIO operations initiated - executed_sync: IOs that were executed synchronously (fallback path) - executed_async: IOs that were submitted asynchronously to the IO method - completed_self: IO completions processed by the issuing backend itself - completed_other: IO completions processed on behalf of another backend - handle_waits: times the backend had to wait for a free AIO handle - submitted: number of submitted calls to the IO method These counters are useful for understanding and tuning AIO behavior: - executed_async / started. A ratio near zero means the backend is falling back to synchronous execution (TOAST chunk fetches, temp buffers, ...). - a non-zero handle_waits means the backend exhausted all its AIO handles. That could mean that io_max_concurrency is too low. - completed_self vs completed_other reveals cross-backend completion patterns. That helps see how IO completion work is distributed and could help interpret per backend IO statistics values. - executed_async / submitted gives the average batch size per submit call. This commit is straight-forward, relying on the infrastructure provided by 9aea73fc61d4 (backend-level pgstats). XXX: Bump catalog version. No need to touch PGSTAT_FILE_FORMAT_ID as backend statistics are never written to disk. Author: Bertrand Drouvot <bertranddrouvot.pg@gmail.com> Reviewed-by: Discussion: --- doc/src/sgml/monitoring.sgml | 70 +++++++++++++ src/backend/storage/aio/aio.c | 13 +++ src/backend/utils/activity/pgstat_backend.c | 109 ++++++++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 60 +++++++++++ src/include/catalog/pg_proc.dat | 7 ++ src/include/pgstat.h | 25 +++++ src/include/utils/pgstat_internal.h | 3 +- src/test/modules/test_aio/t/001_aio.pl | 32 ++++++ src/tools/pgindent/typedefs.list | 1 + 9 files changed, 319 insertions(+), 1 deletion(-) 24.4% doc/src/sgml/ 3.2% src/backend/storage/aio/ 25.3% src/backend/utils/activity/ 20.5% src/backend/utils/adt/ 4.9% src/include/catalog/ 10.0% src/include/ 11.1% src/test/modules/test_aio/t/ diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 12b9ee20d4a..1c566cf0e0e 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -5641,6 +5641,76 @@ description | Waiting for a newly initialized WAL file to reach durable storage </para></entry> </row> + <row> + <entry id="pg-stat-get-backend-aio" role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_stat_get_backend_aio</primary> + </indexterm> + <function>pg_stat_get_backend_aio</function> ( <type>integer</type> ) + <returnvalue>record</returnvalue> + </para> + <para> + Returns <acronym>AIO</acronym> (Asynchronous I/O) statistics about the + backend with the specified process ID. The returned values are: + <itemizedlist> + <listitem> + <para> + <literal>started</literal>: Total IOs initiated. + </para> + </listitem> + <listitem> + <para> + <literal>executed_sync</literal>: IOs executed synchronously. + </para> + </listitem> + <listitem> + <para> + <literal>executed_async</literal>: IOs submitted asynchronously. + </para> + </listitem> + <listitem> + <para> + <literal>completed_self</literal>: IO completions processed by this + backend. + </para> + </listitem> + <listitem> + <para> + <literal>completed_other</literal>: IO completions processed on + behalf of other backends. + </para> + </listitem> + <listitem> + <para> + <literal>handle_waits</literal>: Times waited for a free AIO handle. + </para> + </listitem> + <listitem> + <para> + <literal>submitted</literal>: Number of submit calls to the IO + method. Compare with <literal>executed_async</literal> to determine + average batch size. + </para> + </listitem> + <listitem> + <para> + <literal>stats_reset</literal>: Timestamp of last stats reset. + </para> + </listitem> + </itemizedlist> + </para> + <para> + The <literal>completed_other</literal> column is only meaningful + when <varname>io_method</varname> is set to <literal>io_uring</literal>; + with <literal>worker</literal> mode, IO completions are processed by + IO worker processes which do not track these statistics. + </para> + <para> + The function does not return AIO statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher. + </para></entry> + </row> + <row> <entry role="func_table_entry"><para role="func_signature"> <indexterm> diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 8f7e26607b9..507f1727d41 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -40,6 +40,7 @@ #include "lib/ilist.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/atomics.h" #include "storage/aio.h" #include "storage/aio_internal.h" @@ -458,6 +459,8 @@ pgaio_io_stage(PgAioHandle *ioh, PgAioOp op) "staged (synchronous: %d, in_batch: %d)", needs_synchronous, pgaio_my_backend->in_batchmode); + pgstat_count_backend_aio_start(needs_synchronous); + if (!needs_synchronous) { pgaio_my_backend->staged_ios[pgaio_my_backend->num_staged_ios++] = ioh; @@ -544,6 +547,12 @@ pgaio_io_process_completion(PgAioHandle *ioh, int result) /* condition variable broadcast ensures state is visible before wakeup */ ConditionVariableBroadcast(&ioh->cv); + /* Track AIO completion stats */ + if (ioh->owner_procno == MyProcNumber) + pgstat_count_backend_aio_complete_self(); + else + pgstat_count_backend_aio_complete_other(); + /* contains call to pgaio_io_call_complete_local() */ if (ioh->owner_procno == MyProcNumber) pgaio_io_reclaim(ioh); @@ -762,6 +771,8 @@ pgaio_io_wait_for_free(void) { int reclaimed = 0; + pgstat_count_backend_aio_handle_wait(); + pgaio_debug(DEBUG2, "waiting for free IO with %d pending, %u in-flight, %u idle IOs", pgaio_my_backend->num_staged_ios, dclist_count(&pgaio_my_backend->in_flight_ios), @@ -1150,6 +1161,8 @@ pgaio_submit_staged(void) Assert(total_submitted == did_submit); + pgstat_count_backend_aio_submitted(); + pgaio_my_backend->num_staged_ios = 0; pgaio_debug(DEBUG4, diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index b736b2ccc6f..4ad755f7f18 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -40,6 +40,7 @@ static PgStat_BackendPending PendingBackendStats; static bool backend_has_iostats = false; static bool backend_has_lockstats = false; +static bool backend_has_aiostats = false; /* * WAL usage counters saved from pgWalUsage at the previous call to @@ -120,6 +121,74 @@ pgstat_count_backend_lock_fastpath_exceeded(uint8 locktag_type) pgstat_report_fixed = true; } +/* + * Utility routines to report AIO stats for backends, kept here to avoid + * exposing PendingBackendStats to the outside world. + */ +void +pgstat_count_backend_aio_start(bool synchronous) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.started++; + if (synchronous) + PendingBackendStats.aio_counters.executed_sync++; + else + PendingBackendStats.aio_counters.executed_async++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_complete_self(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.completed_self++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_complete_other(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.completed_other++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_handle_wait(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.handle_waits++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_submitted(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.submitted++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + /* * Returns statistics of a backend by proc number. */ @@ -326,6 +395,38 @@ pgstat_flush_backend_entry_lock(PgStat_EntryRef *entry_ref) backend_has_lockstats = false; } +/* + * Flush out locally pending backend AIO statistics. Locking is managed + * by the caller. + */ +static void +pgstat_flush_backend_entry_aio(PgStat_EntryRef *entry_ref) +{ + PgStatShared_Backend *shbackendent; + PgStat_AioCounters *bktype_shstats; + + if (!backend_has_aiostats) + return; + + shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendent->stats.aio_counters; + +#define AIOSTAT_ACC(fld) \ + (bktype_shstats->fld += PendingBackendStats.aio_counters.fld) + AIOSTAT_ACC(started); + AIOSTAT_ACC(executed_sync); + AIOSTAT_ACC(executed_async); + AIOSTAT_ACC(completed_self); + AIOSTAT_ACC(completed_other); + AIOSTAT_ACC(handle_waits); + AIOSTAT_ACC(submitted); +#undef AIOSTAT_ACC + + MemSet(&PendingBackendStats.aio_counters, 0, sizeof(PgStat_AioCounters)); + + backend_has_aiostats = false; +} + /* * Flush out locally pending backend statistics * @@ -354,6 +455,10 @@ pgstat_flush_backend(bool nowait, uint32 flags) if ((flags & PGSTAT_BACKEND_FLUSH_LOCK) && backend_has_lockstats) has_pending_data = true; + /* Some AIO data pending? */ + if ((flags & PGSTAT_BACKEND_FLUSH_AIO) && backend_has_aiostats) + has_pending_data = true; + if (!has_pending_data) return false; @@ -372,6 +477,9 @@ pgstat_flush_backend(bool nowait, uint32 flags) if (flags & PGSTAT_BACKEND_FLUSH_LOCK) pgstat_flush_backend_entry_lock(entry_ref); + if (flags & PGSTAT_BACKEND_FLUSH_AIO) + pgstat_flush_backend_entry_aio(entry_ref); + pgstat_unlock_entry(entry_ref); return false; @@ -411,6 +519,7 @@ pgstat_create_backend(ProcNumber procnum) MemSet(&PendingBackendStats, 0, sizeof(PgStat_BackendPending)); backend_has_iostats = false; backend_has_lockstats = false; + backend_has_aiostats = false; /* * Initialize prevBackendWalUsage with pgWalUsage so that diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 565d0e70768..fb62a56f3fe 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1722,6 +1722,66 @@ pg_stat_get_backend_wal(PG_FUNCTION_ARGS) return (pg_stat_wal_build_tuple(bktype_stats, backend_stats->stat_reset_timestamp)); } +/* + * Returns AIO statistics for a backend with given PID. + */ +Datum +pg_stat_get_backend_aio(PG_FUNCTION_ARGS) +{ +#define PG_STAT_BACKEND_AIO_COLS 8 + TupleDesc tupdesc; + Datum values[PG_STAT_BACKEND_AIO_COLS] = {0}; + bool nulls[PG_STAT_BACKEND_AIO_COLS] = {0}; + int pid; + PgStat_Backend *backend_stats; + PgStat_AioCounters aio_counters; + + pid = PG_GETARG_INT32(0); + backend_stats = pgstat_fetch_stat_backend_by_pid(pid, NULL); + + if (!backend_stats) + PG_RETURN_NULL(); + + aio_counters = backend_stats->aio_counters; + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_BACKEND_AIO_COLS); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "started", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "executed_sync", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "executed_async", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "completed_self", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "completed_other", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "handle_waits", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "submitted", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "stats_reset", + TIMESTAMPTZOID, -1, 0); + TupleDescFinalize(tupdesc); + BlessTupleDesc(tupdesc); + + /* Fill values */ + values[0] = Int64GetDatum(aio_counters.started); + values[1] = Int64GetDatum(aio_counters.executed_sync); + values[2] = Int64GetDatum(aio_counters.executed_async); + values[3] = Int64GetDatum(aio_counters.completed_self); + values[4] = Int64GetDatum(aio_counters.completed_other); + values[5] = Int64GetDatum(aio_counters.handle_waits); + values[6] = Int64GetDatum(aio_counters.submitted); + + if (backend_stats->stat_reset_timestamp != 0) + values[7] = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + else + nulls[7] = true; + + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 3cb84359adf..a089860ada4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -6108,6 +6108,13 @@ proargmodes => '{i,o,o,o,o,o}', proargnames => '{backend_pid,locktype,waits,wait_time,fastpath_exceeded,stats_reset}', prosrc => 'pg_stat_get_backend_lock' }, +{ oid => '9082', descr => 'statistics: backend AIO activity', + proname => 'pg_stat_get_backend_aio', provolatile => 'v', proparallel => 'r', + prorettype => 'record', proargtypes => 'int4', + proallargtypes => '{int4,int8,int8,int8,int8,int8,int8,int8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,started,executed_sync,executed_async,completed_self,completed_other,handle_waits,submitted,stats_reset}', + prosrc => 'pg_stat_get_backend_aio' }, { oid => '6248', descr => 'statistics: information about WAL prefetching', proname => 'pg_stat_get_recovery_prefetch', prorows => '1', proretset => 't', provolatile => 'v', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 58a44857f13..64afb2bc082 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -514,6 +514,21 @@ typedef struct PgStat_WalStats TimestampTz stat_reset_timestamp; } PgStat_WalStats; +/* ------- + * PgStat_AioCounters AIO activity counters + * ------- + */ +typedef struct PgStat_AioCounters +{ + PgStat_Counter started; + PgStat_Counter executed_sync; + PgStat_Counter executed_async; + PgStat_Counter completed_self; + PgStat_Counter completed_other; + PgStat_Counter handle_waits; + PgStat_Counter submitted; +} PgStat_AioCounters; + /* ------- * PgStat_Backend Backend statistics * ------- @@ -524,6 +539,7 @@ typedef struct PgStat_Backend PgStat_BktypeIO io_stats; PgStat_WalCounters wal_counters; PgStat_PendingLock lock_stats; + PgStat_AioCounters aio_counters; } PgStat_Backend; /* --------- @@ -542,6 +558,8 @@ typedef struct PgStat_BackendPending * PGSTAT_KIND_LOCK. */ PgStat_PendingLock pending_lock; + /* Store the AIO statistics counters */ + PgStat_AioCounters aio_counters; } PgStat_BackendPending; /* @@ -598,6 +616,13 @@ extern void pgstat_count_backend_io_op(IOObject io_object, extern void pgstat_count_backend_lock_waits(uint8 locktag_type, PgStat_Counter usecs); extern void pgstat_count_backend_lock_fastpath_exceeded(uint8 locktag_type); +/* used by aio.c for AIO stats tracked in backends */ +extern void pgstat_count_backend_aio_start(bool synchronous); +extern void pgstat_count_backend_aio_complete_self(void); +extern void pgstat_count_backend_aio_complete_other(void); +extern void pgstat_count_backend_aio_handle_wait(void); +extern void pgstat_count_backend_aio_submitted(void); + extern PgStat_Backend *pgstat_fetch_stat_backend(ProcNumber procNumber); extern PgStat_Backend *pgstat_fetch_stat_backend_by_pid(int pid, BackendType *bktype); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index b3dc3ff7d8b..b2092fb42b9 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -706,7 +706,8 @@ extern void pgstat_archiver_snapshot_cb(void); #define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */ #define PGSTAT_BACKEND_FLUSH_WAL (1 << 1) /* Flush WAL statistics */ #define PGSTAT_BACKEND_FLUSH_LOCK (1 << 2) /* Flush lock statistics */ -#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL | PGSTAT_BACKEND_FLUSH_LOCK) +#define PGSTAT_BACKEND_FLUSH_AIO (1 << 3) /* Flush AIO statistics */ +#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL | PGSTAT_BACKEND_FLUSH_LOCK | PGSTAT_BACKEND_FLUSH_AIO) extern bool pgstat_flush_backend(bool nowait, uint32 flags); extern bool pgstat_backend_flush_cb(bool nowait); diff --git a/src/test/modules/test_aio/t/001_aio.pl b/src/test/modules/test_aio/t/001_aio.pl index 63cadd64c15..8ecf3d3ad91 100644 --- a/src/test/modules/test_aio/t/001_aio.pl +++ b/src/test/modules/test_aio/t/001_aio.pl @@ -1842,6 +1842,37 @@ read_buffers('$table', 0, 4)|, $psql_c->quit(); } +# Test per-backend AIO statistics counters +sub test_aio_stats +{ + my $io_method = shift; + my $node = shift; + + my $psql = $node->background_psql('postgres', on_error_stop => 0); + + # Reset backend stats, evict relation, then read it back to force + # physical IO through the AIO layer. + $psql->query_safe(qq(SELECT pg_stat_reset_backend_stats(pg_backend_pid()))); + $psql->query_safe(qq(SELECT evict_rel('tbl_ok'))); + $psql->query_safe(qq(SELECT count(*) FROM tbl_ok)); + $psql->query_safe(qq(SELECT pg_stat_force_next_flush())); + + # started must be > 0 after physical IO + my $started = $psql->query_safe( + qq(SELECT started FROM pg_stat_get_backend_aio(pg_backend_pid()))); + cmp_ok($started, '>', 0, + "$io_method: AIO stats: started > 0 after physical IO"); + + # invariant: started = executed_sync + executed_async + my $consistent = $psql->query_safe( + qq(SELECT started = executed_sync + executed_async + FROM pg_stat_get_backend_aio(pg_backend_pid()))); + is($consistent, 't', + "$io_method: AIO stats: started = executed_sync + executed_async"); + + $psql->quit(); +} + # Run all tests that for the specified node / io_method sub test_io_method { @@ -1878,6 +1909,7 @@ CHECKPOINT; test_ignore_checksum($io_method, $node); test_checksum_createdb($io_method, $node); test_read_buffers($io_method, $node); + test_aio_stats($io_method, $node); # generic injection tests SKIP: diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 117e7379f10..6ce3077fa5b 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2329,6 +2329,7 @@ PgStatShared_ReplSlot PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal +PgStat_AioCounters PgStat_ArchiverStats PgStat_Backend PgStat_BackendPending -- 2.34.1 --Pi0/Nu3LVhi0Ij+H-- ^ permalink raw reply [nested|flat] 3+ messages in thread
end of thread, other threads:[~2026-06-11 09:45 UTC | newest] Thread overview: 3+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-12 21:35 [PATCH 3/6] Default to zstd.. Justin Pryzby <pryzbyj@telsasoft.com> 2021-03-12 21:35 [PATCH 08/10] Default to zstd.. Justin Pryzby <pryzbyj@telsasoft.com> 2026-06-11 09:45 [PATCH v1] Add per-backend AIO statistics Bertrand Drouvot <bertranddrouvot.pg@gmail.com>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox