public inbox for [email protected]help / color / mirror / Atom feed
[PATCH 5/8] f!2nd attempt to use pkgconfig to allow compiling on OSX 2+ messages / 2 participants [nested] [flat]
* [PATCH 5/8] f!2nd attempt to use pkgconfig to allow compiling on OSX @ 2021-03-11 15:18 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: Justin Pryzby @ 2021-03-11 15:18 UTC (permalink / raw) --- configure | 279 +++++++++++++++++++-------------------------------- configure.ac | 16 ++- 2 files changed, 108 insertions(+), 187 deletions(-) diff --git a/configure b/configure index 87ed16060e..387c553d5a 100755 --- a/configure +++ b/configure @@ -654,8 +654,6 @@ UUID_LIBS LDAP_LIBS_BE LDAP_LIBS_FE with_ssl -LZ4_LIBS -LZ4_CFLAGS PTHREAD_CFLAGS PTHREAD_LIBS PTHREAD_CC @@ -701,6 +699,8 @@ with_gnu_ld LD LDFLAGS_SL LDFLAGS_EX +LZ4_LIBS +LZ4_CFLAGS with_lz4 with_zlib with_system_tzdata @@ -895,12 +895,12 @@ ICU_LIBS XML2_CONFIG XML2_CFLAGS XML2_LIBS +LZ4_CFLAGS +LZ4_LIBS LDFLAGS_EX LDFLAGS_SL PERL PYTHON -LZ4_CFLAGS -LZ4_LIBS MSGFMT TCLSH' @@ -1603,12 +1603,12 @@ Some influential environment variables: XML2_CONFIG path to xml2-config utility XML2_CFLAGS C compiler flags for XML2, overriding pkg-config XML2_LIBS linker flags for XML2, overriding pkg-config + LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config + LZ4_LIBS linker flags for LZ4, overriding pkg-config LDFLAGS_EX extra linker flags for linking executables only LDFLAGS_SL extra linker flags for linking shared libraries only PERL Perl program PYTHON Python program - LZ4_CFLAGS C compiler flags for LZ4, overriding pkg-config - LZ4_LIBS linker flags for LZ4, overriding pkg-config MSGFMT msgfmt program for NLS TCLSH Tcl interpreter program (tclsh) @@ -8607,6 +8607,102 @@ fi $as_echo "$with_lz4" >&6; } +if test "$with_lz4" = yes; then + +pkg_failed=no +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 +$as_echo_n "checking for liblz4... " >&6; } + +if test -n "$LZ4_CFLAGS"; then + pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$LZ4_LIBS"; then + pkg_cv_LZ4_LIBS="$LZ4_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 + ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 + ac_status=$? + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` + else + LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$LZ4_PKG_ERRORS" >&5 + + as_fn_error $? "Package requirements (liblz4) were not met: + +$LZ4_PKG_ERRORS + +Consider adjusting the PKG_CONFIG_PATH environment variable if you +installed software in a non-standard prefix. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details." "$LINENO" 5 +elif test $pkg_failed = untried; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it +is in your PATH or set the PKG_CONFIG environment variable to the full +path to pkg-config. + +Alternatively, you may set the environment variables LZ4_CFLAGS +and LZ4_LIBS to avoid the need to call pkg-config. +See the pkg-config man page for more details. + +To get pkg-config, see <http://pkg-config.freedesktop.org/;. +See \`config.log' for more details" "$LINENO" 5; } +else + LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS + LZ4_LIBS=$pkg_cv_LZ4_LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +fi + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -12154,147 +12250,6 @@ fi fi -if test "$with_lz4" = yes; then - -pkg_failed=no -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for liblz4" >&5 -$as_echo_n "checking for liblz4... " >&6; } - -if test -n "$LZ4_CFLAGS"; then - pkg_cv_LZ4_CFLAGS="$LZ4_CFLAGS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 - ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_LZ4_CFLAGS=`$PKG_CONFIG --cflags "liblz4" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi -if test -n "$LZ4_LIBS"; then - pkg_cv_LZ4_LIBS="$LZ4_LIBS" - elif test -n "$PKG_CONFIG"; then - if test -n "$PKG_CONFIG" && \ - { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"liblz4\""; } >&5 - ($PKG_CONFIG --exists --print-errors "liblz4") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then - pkg_cv_LZ4_LIBS=`$PKG_CONFIG --libs "liblz4" 2>/dev/null` - test "x$?" != "x0" && pkg_failed=yes -else - pkg_failed=yes -fi - else - pkg_failed=untried -fi - - - -if test $pkg_failed = yes; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - -if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then - _pkg_short_errors_supported=yes -else - _pkg_short_errors_supported=no -fi - if test $_pkg_short_errors_supported = yes; then - LZ4_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "liblz4" 2>&1` - else - LZ4_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "liblz4" 2>&1` - fi - # Put the nasty error message in config.log where it belongs - echo "$LZ4_PKG_ERRORS" >&5 - - as_fn_error $? "Package requirements (liblz4) were not met: - -$LZ4_PKG_ERRORS - -Consider adjusting the PKG_CONFIG_PATH environment variable if you -installed software in a non-standard prefix. - -Alternatively, you may set the environment variables LZ4_CFLAGS -and LZ4_LIBS to avoid the need to call pkg-config. -See the pkg-config man page for more details." "$LINENO" 5 -elif test $pkg_failed = untried; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error $? "The pkg-config script could not be found or is too old. Make sure it -is in your PATH or set the PKG_CONFIG environment variable to the full -path to pkg-config. - -Alternatively, you may set the environment variables LZ4_CFLAGS -and LZ4_LIBS to avoid the need to call pkg-config. -See the pkg-config man page for more details. - -To get pkg-config, see <http://pkg-config.freedesktop.org/;. -See \`config.log' for more details" "$LINENO" 5; } -else - LZ4_CFLAGS=$pkg_cv_LZ4_CFLAGS - LZ4_LIBS=$pkg_cv_LZ4_LIBS - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - -fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4_compress in -llz4" >&5 -$as_echo_n "checking for LZ4_compress in -llz4... " >&6; } -if ${ac_cv_lib_lz4_LZ4_compress+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_check_lib_save_LIBS=$LIBS -LIBS="-llz4 $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char LZ4_compress (); -int -main () -{ -return LZ4_compress (); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_lz4_LZ4_compress=yes -else - ac_cv_lib_lz4_LZ4_compress=no -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lz4_LZ4_compress" >&5 -$as_echo "$ac_cv_lib_lz4_LZ4_compress" >&6; } -if test "x$ac_cv_lib_lz4_LZ4_compress" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LIBLZ4 1 -_ACEOF - - LIBS="-llz4 $LIBS" - -else - as_fn_error $? "library 'lz4' is required for LZ4 support" "$LINENO" 5 -fi - -fi - if test "$enable_spinlocks" = yes; then $as_echo "#define HAVE_SPINLOCKS 1" >>confdefs.h @@ -13561,36 +13516,6 @@ Use --without-zlib to disable zlib support." "$LINENO" 5 fi -fi - -if test "$with_lz4" = yes; then - for ac_header in lz4/lz4.h -do : - ac_fn_c_check_header_mongrel "$LINENO" "lz4/lz4.h" "ac_cv_header_lz4_lz4_h" "$ac_includes_default" -if test "x$ac_cv_header_lz4_lz4_h" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LZ4_LZ4_H 1 -_ACEOF - -else - for ac_header in lz4.h -do : - ac_fn_c_check_header_mongrel "$LINENO" "lz4.h" "ac_cv_header_lz4_h" "$ac_includes_default" -if test "x$ac_cv_header_lz4_h" = xyes; then : - cat >>confdefs.h <<_ACEOF -#define HAVE_LZ4_H 1 -_ACEOF - -else - as_fn_error $? "lz4.h header file is required for LZ4" "$LINENO" 5 -fi - -done - -fi - -done - fi if test "$with_gssapi" = yes ; then diff --git a/configure.ac b/configure.ac index c8f199dc5e..4cd683afb3 100644 --- a/configure.ac +++ b/configure.ac @@ -995,6 +995,12 @@ PGAC_ARG_BOOL(with, lz4, no, [build with LZ4 support], AC_MSG_RESULT([$with_lz4]) AC_SUBST(with_lz4) +if test "$with_lz4" = yes; then + PKG_CHECK_MODULES(LZ4, liblz4) + LIBS="$LZ4_LIBS $LIBS" + CFLAGS="$LZ4_CFLAGS $CFLAGS" +fi + # # Assignments # @@ -1183,11 +1189,6 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi -if test "$with_lz4" = yes; then - PKG_CHECK_MODULES(LZ4, liblz4) - AC_CHECK_LIB(lz4, LZ4_compress, [], [AC_MSG_ERROR([library 'lz4' is required for LZ4 support])]) -fi - if test "$enable_spinlocks" = yes; then AC_DEFINE(HAVE_SPINLOCKS, 1, [Define to 1 if you have spinlocks.]) else @@ -1421,11 +1422,6 @@ failure. It is possible the compiler isn't looking in the proper directory. Use --without-zlib to disable zlib support.])]) fi -if test "$with_lz4" = yes; then - AC_CHECK_HEADERS(lz4/lz4.h, [], - [AC_CHECK_HEADERS(lz4.h, [], [AC_MSG_ERROR([lz4.h header file is required for LZ4])])]) -fi - if test "$with_gssapi" = yes ; then AC_CHECK_HEADERS(gssapi/gssapi.h, [], [AC_CHECK_HEADERS(gssapi.h, [], [AC_MSG_ERROR([gssapi.h header file is required for GSSAPI])])]) -- 2.17.0 --cvVnyQ+4j833TQvp Content-Type: text/x-diff; charset=us-ascii Content-Disposition: attachment; filename="0006-Add-default_toast_compression-GUC.patch" ^ permalink raw reply [nested|flat] 2+ messages in thread
* Re: [PATCH] Add extra statistics to explain for Nested Loop @ 2022-04-02 12:38 Justin Pryzby <[email protected]> 0 siblings, 0 replies; 2+ messages in thread From: Justin Pryzby @ 2022-04-02 12:38 UTC (permalink / raw) To: Ekaterina Sokolova <[email protected]>; +Cc: Julien Rouhaud <[email protected]>; Greg Stark <[email protected]>; Lukas Fittl <[email protected]>; [email protected] This message lost track of the email headers so CFBOT isn't processing the new patches. Which I'm attempting to remedy now. https://www.postgresql.org/message-id/flat/[email protected] On Fri, Apr 01, 2022 at 11:46:47PM +0300, Ekaterina Sokolova wrote: > Hi, hackers. Thank you for your attention to this topic. > > Julien Rouhaud wrote: > > +static void show_loop_info(Instrumentation *instrument, bool isworker, > > + ExplainState *es); > > > > I think this should be done as a separate refactoring commit. > Sure. I divided the patch. Now Justin's refactor commit is separated. Also I > actualized it a bit. > > > Most of the comments I have are easy to fix. But I think that the real > > problem > > is the significant overhead shown by Ekaterina that for now would apply > > even if > > you don't consume the new stats, for instance if you have > > pg_stat_statements. > > And I'm still not sure of what is the best way to avoid that. > I took your advice about InstrumentOption. Now INSTRUMENT_EXTRA exists. > So currently it's no overheads during basic load. Operations using > INSTRUMENT_ALL contain overheads (because of INSTRUMENT_EXTRA is a part of > INSTRUMENT_ALL), but they are much less significant than before. I apply new > overhead statistics collected by pgbench with auto _explain enabled. > > > Why do you need to initialize min_t and min_tuples but not max_t and > > max_tuples while both will initially be 0 and possibly updated > > afterwards? > We need this initialization for min values so comment about it located above > the block of code with initialization. > > I am convinced that the latest changes have affected the patch in a positive > way. I'll be pleased to hear your thoughts on this. Attachments: [text/x-diff] 0001-explain.c-refactor-ExplainNode.patch (4.9K, ../../[email protected]/2-0001-explain.c-refactor-ExplainNode.patch) download | inline diff: From 0dec500a0ed934d5d2038cb087ba6a605cafcdef Mon Sep 17 00:00:00 2001 From: Justin Pryzby <[email protected]> Date: Thu, 15 Apr 2021 11:55:09 -0500 Subject: [PATCH 1/2] explain.c: refactor ExplainNode() --- src/backend/commands/explain.c | 110 ++++++++++++++------------------- 1 file changed, 47 insertions(+), 63 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index cb13227db1f..06e089a1220 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -118,6 +118,8 @@ static void show_instrumentation_count(const char *qlabel, int which, PlanState *planstate, ExplainState *es); static void show_foreignscan_info(ForeignScanState *fsstate, ExplainState *es); static void show_eval_params(Bitmapset *bms_params, ExplainState *es); +static void show_loop_info(Instrumentation *instrument, bool isworker, + ExplainState *es); static const char *explain_get_index_name(Oid indexId); static void show_buffer_usage(ExplainState *es, const BufferUsage *usage, bool planning); @@ -1615,36 +1617,7 @@ ExplainNode(PlanState *planstate, List *ancestors, if (es->analyze && planstate->instrument && planstate->instrument->nloops > 0) - { - double nloops = planstate->instrument->nloops; - double startup_ms = 1000.0 * planstate->instrument->startup / nloops; - double total_ms = 1000.0 * planstate->instrument->total / nloops; - double rows = planstate->instrument->ntuples / nloops; - - if (es->format == EXPLAIN_FORMAT_TEXT) - { - if (es->timing) - appendStringInfo(es->str, - " (actual time=%.3f..%.3f rows=%.0f loops=%.0f)", - startup_ms, total_ms, rows, nloops); - else - appendStringInfo(es->str, - " (actual rows=%.0f loops=%.0f)", - rows, nloops); - } - else - { - if (es->timing) - { - ExplainPropertyFloat("Actual Startup Time", "ms", startup_ms, - 3, es); - ExplainPropertyFloat("Actual Total Time", "ms", total_ms, - 3, es); - } - ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); - ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); - } - } + show_loop_info(planstate->instrument, false, es); else if (es->analyze) { if (es->format == EXPLAIN_FORMAT_TEXT) @@ -1673,44 +1646,14 @@ ExplainNode(PlanState *planstate, List *ancestors, for (int n = 0; n < w->num_workers; n++) { Instrumentation *instrument = &w->instrument[n]; - double nloops = instrument->nloops; - double startup_ms; - double total_ms; - double rows; - if (nloops <= 0) + if (instrument->nloops <= 0) continue; - startup_ms = 1000.0 * instrument->startup / nloops; - total_ms = 1000.0 * instrument->total / nloops; - rows = instrument->ntuples / nloops; ExplainOpenWorker(n, es); - + show_loop_info(instrument, true, es); if (es->format == EXPLAIN_FORMAT_TEXT) - { - ExplainIndentText(es); - if (es->timing) - appendStringInfo(es->str, - "actual time=%.3f..%.3f rows=%.0f loops=%.0f\n", - startup_ms, total_ms, rows, nloops); - else - appendStringInfo(es->str, - "actual rows=%.0f loops=%.0f\n", - rows, nloops); - } - else - { - if (es->timing) - { - ExplainPropertyFloat("Actual Startup Time", "ms", - startup_ms, 3, es); - ExplainPropertyFloat("Actual Total Time", "ms", - total_ms, 3, es); - } - ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); - ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); - } - + appendStringInfoChar(es->str, '\n'); ExplainCloseWorker(n, es); } } @@ -4039,6 +3982,47 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, ExplainCloseGroup("Target Tables", "Target Tables", false, es); } +void +show_loop_info(Instrumentation *instrument, bool isworker, ExplainState *es) +{ + double nloops = instrument->nloops; + double startup_ms = 1000.0 * instrument->startup / nloops; + double total_ms = 1000.0 * instrument->total / nloops; + double rows = instrument->ntuples / nloops; + + if (es->format == EXPLAIN_FORMAT_TEXT) + { + if (isworker) + ExplainIndentText(es); + else + appendStringInfo(es->str, " ("); + + if (es->timing) + appendStringInfo(es->str, + "actual time=%.3f..%.3f rows=%.0f loops=%.0f", + startup_ms, total_ms, rows, nloops); + else + appendStringInfo(es->str, + "actual rows=%.0f loops=%.0f", + rows, nloops); + + if (!isworker) + appendStringInfoChar(es->str, ')'); + } + else + { + if (es->timing) + { + ExplainPropertyFloat("Actual Startup Time", "ms", startup_ms, + 3, es); + ExplainPropertyFloat("Actual Total Time", "ms", total_ms, + 3, es); + } + ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); + } +} + /* * Explain the constituent plans of an Append, MergeAppend, * BitmapAnd, or BitmapOr node. -- 2.17.1 [text/x-diff] 0002-Add-extra-statistics-to-explain-for-Nested-Loop.patch (19.6K, ../../[email protected]/3-0002-Add-extra-statistics-to-explain-for-Nested-Loop.patch) download | inline diff: From e0d92f2c112002bbacaae7abb747ea618d7fbd93 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova <[email protected]> Date: Fri, 1 Apr 2022 21:36:21 -0500 Subject: [PATCH 2/2] Add extra statistics to explain for Nested Loop For some distributions of data in tables, different loops in nested loop joins can take different time and process different amounts of entries. It makes average statistics returned by explain analyze not very useful for DBA. This patch add collecting of min, max and total statistics for time and rows across all loops to EXPLAIN ANALYSE. You need to set the VERBOSE flag to display this information. The patch contains regression tests. Example of results in TEXT format: -> Append (actual rows=5 loops=5) Loop Min Rows: 2 Max Rows: 6 Total Rows: 23 Reviewed-by: Lukas Fittl, Justin Pryzby, Yugo Nagata, Julien Rouhaud. --- src/backend/commands/explain.c | 42 ++++++- src/backend/executor/instrument.c | 38 +++++- src/include/executor/instrument.h | 10 ++ src/test/regress/expected/explain.out | 10 ++ src/test/regress/expected/partition_prune.out | 117 ++++++++++++++++++ src/test/regress/sql/partition_prune.sql | 32 +++++ 6 files changed, 247 insertions(+), 2 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 06e089a1220..df14803ff06 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -119,7 +119,7 @@ static void show_instrumentation_count(const char *qlabel, int which, static void show_foreignscan_info(ForeignScanState *fsstate, ExplainState *es); static void show_eval_params(Bitmapset *bms_params, ExplainState *es); static void show_loop_info(Instrumentation *instrument, bool isworker, - ExplainState *es); + ExplainState *es); static const char *explain_get_index_name(Oid indexId); static void show_buffer_usage(ExplainState *es, const BufferUsage *usage, bool planning); @@ -541,6 +541,9 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, if (es->wal) instrument_option |= INSTRUMENT_WAL; + if (es->verbose) + instrument_option |= INSTRUMENT_EXTRA; + /* * We always collect timing for the entire statement, even when node-level * timing is off, so we don't look at es->timing here. (We could skip @@ -1624,6 +1627,7 @@ ExplainNode(PlanState *planstate, List *ancestors, appendStringInfoString(es->str, " (never executed)"); else { + /* without min and max values because actual result is 0 */ if (es->timing) { ExplainPropertyFloat("Actual Startup Time", "ms", 0.0, 3, es); @@ -3989,6 +3993,11 @@ show_loop_info(Instrumentation *instrument, bool isworker, ExplainState *es) double startup_ms = 1000.0 * instrument->startup / nloops; double total_ms = 1000.0 * instrument->total / nloops; double rows = instrument->ntuples / nloops; + double loop_total_rows = instrument->ntuples; + double loop_min_r = instrument->min_tuples; + double loop_max_r = instrument->max_tuples; + double loop_min_t_ms = 1000.0 * instrument->min_t; + double loop_max_t_ms = 1000.0 * instrument->max_t; if (es->format == EXPLAIN_FORMAT_TEXT) { @@ -4008,6 +4017,21 @@ show_loop_info(Instrumentation *instrument, bool isworker, ExplainState *es) if (!isworker) appendStringInfoChar(es->str, ')'); + + if (nloops > 1 && es->verbose) + { + appendStringInfoChar(es->str, '\n'); + ExplainIndentText(es); + + if (es->timing) + appendStringInfo(es->str, + "Loop Min Rows: %.0f Max Rows: %.0f Total Rows: %.0f Min Time: %.3f Max Time: %.3f", + loop_min_r, loop_max_r, loop_total_rows, loop_min_t_ms, loop_max_t_ms); + else + appendStringInfo(es->str, + "Loop Min Rows: %.0f Max Rows: %.0f Total Rows: %.0f", + loop_min_r, loop_max_r, loop_total_rows); + } } else { @@ -4017,8 +4041,24 @@ show_loop_info(Instrumentation *instrument, bool isworker, ExplainState *es) 3, es); ExplainPropertyFloat("Actual Total Time", "ms", total_ms, 3, es); + if (nloops > 1 && es->verbose) + { + ExplainPropertyFloat("Loop Min Time", "ms", loop_min_t_ms, + 3, es); + ExplainPropertyFloat("Loop Max Time", "ms", loop_max_t_ms, + 3, es); + } } + ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); + + if (nloops > 1 && es->verbose) + { + ExplainPropertyFloat("Loop Min Rows", NULL, loop_min_r, 0, es); + ExplainPropertyFloat("Loop Max Rows", NULL, loop_max_r, 0, es); + ExplainPropertyFloat("Loop Total Rows", NULL, loop_total_rows, 0, es); + } + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); } } diff --git a/src/backend/executor/instrument.c b/src/backend/executor/instrument.c index c5ff02a8424..ac46011e516 100644 --- a/src/backend/executor/instrument.c +++ b/src/backend/executor/instrument.c @@ -34,11 +34,12 @@ InstrAlloc(int n, int instrument_options, bool async_mode) /* initialize all fields to zeroes, then modify as needed */ instr = palloc0(n * sizeof(Instrumentation)); - if (instrument_options & (INSTRUMENT_BUFFERS | INSTRUMENT_TIMER | INSTRUMENT_WAL)) + if (instrument_options & (INSTRUMENT_BUFFERS | INSTRUMENT_TIMER | INSTRUMENT_WAL | INSTRUMENT_EXTRA)) { bool need_buffers = (instrument_options & INSTRUMENT_BUFFERS) != 0; bool need_wal = (instrument_options & INSTRUMENT_WAL) != 0; bool need_timer = (instrument_options & INSTRUMENT_TIMER) != 0; + bool need_extra = (instrument_options & INSTRUMENT_EXTRA) != 0; int i; for (i = 0; i < n; i++) @@ -46,6 +47,7 @@ InstrAlloc(int n, int instrument_options, bool async_mode) instr[i].need_bufusage = need_buffers; instr[i].need_walusage = need_wal; instr[i].need_timer = need_timer; + instr[i].need_extra = need_extra; instr[i].async_mode = async_mode; } } @@ -61,6 +63,7 @@ InstrInit(Instrumentation *instr, int instrument_options) instr->need_bufusage = (instrument_options & INSTRUMENT_BUFFERS) != 0; instr->need_walusage = (instrument_options & INSTRUMENT_WAL) != 0; instr->need_timer = (instrument_options & INSTRUMENT_TIMER) != 0; + instr->need_extra = (instrument_options & INSTRUMENT_EXTRA) != 0; } /* Entry to a plan node */ @@ -154,6 +157,35 @@ InstrEndLoop(Instrumentation *instr) instr->startup += instr->firsttuple; instr->total += totaltime; instr->ntuples += instr->tuplecount; + + /* + * this is first loop + * + * We only initialize the min values. We don't need to bother with the + * max, because those are 0 and the non-zero values will get updated a + * couple lines later. + */ + if (instr->need_extra) + { + if (instr->nloops == 0) + { + instr->min_t = totaltime; + instr->min_tuples = instr->tuplecount; + } + + if (instr->min_t > totaltime) + instr->min_t = totaltime; + + if (instr->max_t < totaltime) + instr->max_t = totaltime; + + if (instr->min_tuples > instr->tuplecount) + instr->min_tuples = instr->tuplecount; + + if (instr->max_tuples < instr->tuplecount) + instr->max_tuples = instr->tuplecount; + } + instr->nloops += 1; /* Reset for next cycle (if any) */ @@ -186,6 +218,10 @@ InstrAggNode(Instrumentation *dst, Instrumentation *add) dst->nloops += add->nloops; dst->nfiltered1 += add->nfiltered1; dst->nfiltered2 += add->nfiltered2; + dst->min_t = Min(dst->min_t, add->min_t); + dst->max_t = Max(dst->max_t, add->max_t); + dst->min_tuples = Min(dst->min_tuples, add->min_tuples); + dst->max_tuples = Max(dst->max_tuples, add->max_tuples); /* Add delta of buffer usage since entry to node's totals */ if (dst->need_bufusage) diff --git a/src/include/executor/instrument.h b/src/include/executor/instrument.h index 1b7157bdd15..e6178e248dc 100644 --- a/src/include/executor/instrument.h +++ b/src/include/executor/instrument.h @@ -58,6 +58,8 @@ typedef enum InstrumentOption INSTRUMENT_BUFFERS = 1 << 1, /* needs buffer usage */ INSTRUMENT_ROWS = 1 << 2, /* needs row count */ INSTRUMENT_WAL = 1 << 3, /* needs WAL usage */ + INSTRUMENT_EXTRA = 1 << 4, /* needs counters for min, + * max and total values */ INSTRUMENT_ALL = PG_INT32_MAX } InstrumentOption; @@ -67,6 +69,8 @@ typedef struct Instrumentation bool need_timer; /* true if we need timer data */ bool need_bufusage; /* true if we need buffer usage data */ bool need_walusage; /* true if we need WAL usage data */ + bool need_extra; /* true if we need min, max and total + * statistics for loops */ bool async_mode; /* true if node is in async mode */ /* Info about current plan cycle: */ bool running; /* true if we've completed first tuple */ @@ -79,7 +83,13 @@ typedef struct Instrumentation /* Accumulated statistics across all completed cycles: */ double startup; /* total startup time (in seconds) */ double total; /* total time (in seconds) */ + double min_t; /* time of fastest loop (in seconds) */ + double max_t; /* time of slowest loop (in seconds) */ double ntuples; /* total tuples produced */ + double min_tuples; /* min counter of produced tuples for all + * loops */ + double max_tuples; /* max counter of produced tuples for all + * loops */ double ntuples2; /* secondary node-specific tuple counter */ double nloops; /* # of run cycles for this node */ double nfiltered1; /* # of tuples removed by scanqual or joinqual */ diff --git a/src/test/regress/expected/explain.out b/src/test/regress/expected/explain.out index bc361759219..c70d0e288da 100644 --- a/src/test/regress/expected/explain.out +++ b/src/test/regress/expected/explain.out @@ -357,8 +357,13 @@ select jsonb_pretty( "Actual Loops": 0, + "Startup Cost": 0.0, + "Async Capable": false, + + "Loop Max Rows": 0, + + "Loop Max Time": 0.0, + + "Loop Min Rows": 0, + + "Loop Min Time": 0.0, + "Relation Name": "tenk1", + "Parallel Aware": true, + + "Loop Total Rows": 0, + "Local Hit Blocks": 0, + "Temp Read Blocks": 0, + "Actual Total Time": 0.0, + @@ -403,7 +408,12 @@ select jsonb_pretty( "Actual Loops": 0, + "Startup Cost": 0.0, + "Async Capable": false, + + "Loop Max Rows": 0, + + "Loop Max Time": 0.0, + + "Loop Min Rows": 0, + + "Loop Min Time": 0.0, + "Parallel Aware": false, + + "Loop Total Rows": 0, + "Sort Space Used": 0, + "Local Hit Blocks": 0, + "Temp Read Blocks": 0, + diff --git a/src/test/regress/expected/partition_prune.out b/src/test/regress/expected/partition_prune.out index 7555764c779..18617b9e206 100644 --- a/src/test/regress/expected/partition_prune.out +++ b/src/test/regress/expected/partition_prune.out @@ -1962,6 +1962,24 @@ begin end loop; end; $$; +create function explain_verbose_parallel_append(text) returns setof text +language plpgsql as +$$ +declare + ln text; +begin + for ln in + execute format('explain (analyze, verbose, costs off, summary off, timing off) %s', + $1) + loop + ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N'); + ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N'); + ln := regexp_replace(ln, 'Loop Min Rows: \d+ Max Rows: \d+ Total Rows: \d+', + 'Loop Min Rows: N Max Rows: N Total Rows: N'); + return next ln; + end loop; +end; +$$; prepare ab_q4 (int, int) as select avg(a) from ab where a between $1 and $2 and b < 4; -- Encourage use of parallel plans @@ -2218,6 +2236,39 @@ select explain_parallel_append('select avg(ab.a) from ab inner join lprt_a a on Index Cond: (a = a.a) (28 rows) +-- Tests for extra statistics +create table lprt_b (b int not null); +insert into lprt_b select generate_series(1,20); +select explain_verbose_parallel_append('select * from lprt_a join lprt_b on a != b'); + explain_verbose_parallel_append +------------------------------------------------------------------------------ + Nested Loop (actual rows=N loops=N) + Output: lprt_a.a, lprt_b.b + Join Filter: (lprt_a.a <> lprt_b.b) + Rows Removed by Join Filter: 4 + -> Gather (actual rows=N loops=N) + Output: lprt_b.b + Workers Planned: 2 + Workers Launched: N + -> Parallel Seq Scan on public.lprt_b (actual rows=N loops=N) + Loop Min Rows: N Max Rows: N Total Rows: N + Output: lprt_b.b + Worker 0: actual rows=N loops=N + Worker 1: actual rows=N loops=N + -> Materialize (actual rows=N loops=N) + Loop Min Rows: N Max Rows: N Total Rows: N + Output: lprt_a.a + -> Gather (actual rows=N loops=N) + Output: lprt_a.a + Workers Planned: 1 + Workers Launched: N + -> Parallel Seq Scan on public.lprt_a (actual rows=N loops=N) + Loop Min Rows: N Max Rows: N Total Rows: N + Output: lprt_a.a + Worker 0: actual rows=N loops=N +(24 rows) + +drop table lprt_b; delete from lprt_a where a = 1; select explain_parallel_append('select avg(ab.a) from ab inner join lprt_a a on ab.a = a.a where a.a in(1, 0, 0)'); explain_parallel_append @@ -2715,6 +2766,72 @@ order by tbl1.col1, tprt.col1; 1001 | 1001 (3 rows) +-- Tests for extra statistics +explain (analyze, verbose, costs off, summary off, timing off) +select * from tbl1 inner join tprt on tbl1.col1 > tprt.col1; + QUERY PLAN +--------------------------------------------------------------------------------- + Nested Loop (actual rows=23 loops=1) + Output: tbl1.col1, tprt.col1 + -> Seq Scan on public.tbl1 (actual rows=5 loops=1) + Output: tbl1.col1 + -> Append (actual rows=5 loops=5) + Loop Min Rows: 2 Max Rows: 6 Total Rows: 23 + -> Index Scan using tprt1_idx on public.tprt_1 (actual rows=2 loops=5) + Loop Min Rows: 2 Max Rows: 2 Total Rows: 10 + Output: tprt_1.col1 + Index Cond: (tprt_1.col1 < tbl1.col1) + -> Index Scan using tprt2_idx on public.tprt_2 (actual rows=3 loops=4) + Loop Min Rows: 2 Max Rows: 3 Total Rows: 11 + Output: tprt_2.col1 + Index Cond: (tprt_2.col1 < tbl1.col1) + -> Index Scan using tprt3_idx on public.tprt_3 (actual rows=1 loops=2) + Loop Min Rows: 1 Max Rows: 1 Total Rows: 2 + Output: tprt_3.col1 + Index Cond: (tprt_3.col1 < tbl1.col1) + -> Index Scan using tprt4_idx on public.tprt_4 (never executed) + Output: tprt_4.col1 + Index Cond: (tprt_4.col1 < tbl1.col1) + -> Index Scan using tprt5_idx on public.tprt_5 (never executed) + Output: tprt_5.col1 + Index Cond: (tprt_5.col1 < tbl1.col1) + -> Index Scan using tprt6_idx on public.tprt_6 (never executed) + Output: tprt_6.col1 + Index Cond: (tprt_6.col1 < tbl1.col1) +(27 rows) + +explain (analyze, verbose, costs off, summary off, timing off) +select * from tbl1 inner join tprt on tbl1.col1 = tprt.col1; + QUERY PLAN +--------------------------------------------------------------------------------- + Nested Loop (actual rows=3 loops=1) + Output: tbl1.col1, tprt.col1 + -> Seq Scan on public.tbl1 (actual rows=5 loops=1) + Output: tbl1.col1 + -> Append (actual rows=1 loops=5) + Loop Min Rows: 0 Max Rows: 1 Total Rows: 3 + -> Index Scan using tprt1_idx on public.tprt_1 (never executed) + Output: tprt_1.col1 + Index Cond: (tprt_1.col1 = tbl1.col1) + -> Index Scan using tprt2_idx on public.tprt_2 (actual rows=1 loops=2) + Loop Min Rows: 1 Max Rows: 1 Total Rows: 2 + Output: tprt_2.col1 + Index Cond: (tprt_2.col1 = tbl1.col1) + -> Index Scan using tprt3_idx on public.tprt_3 (actual rows=0 loops=3) + Loop Min Rows: 0 Max Rows: 1 Total Rows: 1 + Output: tprt_3.col1 + Index Cond: (tprt_3.col1 = tbl1.col1) + -> Index Scan using tprt4_idx on public.tprt_4 (never executed) + Output: tprt_4.col1 + Index Cond: (tprt_4.col1 = tbl1.col1) + -> Index Scan using tprt5_idx on public.tprt_5 (never executed) + Output: tprt_5.col1 + Index Cond: (tprt_5.col1 = tbl1.col1) + -> Index Scan using tprt6_idx on public.tprt_6 (never executed) + Output: tprt_6.col1 + Index Cond: (tprt_6.col1 = tbl1.col1) +(26 rows) + -- Last partition delete from tbl1; insert into tbl1 values (4400); diff --git a/src/test/regress/sql/partition_prune.sql b/src/test/regress/sql/partition_prune.sql index d70bd8610cb..68110e20eeb 100644 --- a/src/test/regress/sql/partition_prune.sql +++ b/src/test/regress/sql/partition_prune.sql @@ -469,6 +469,25 @@ begin end; $$; +create function explain_verbose_parallel_append(text) returns setof text +language plpgsql as +$$ +declare + ln text; +begin + for ln in + execute format('explain (analyze, verbose, costs off, summary off, timing off) %s', + $1) + loop + ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N'); + ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N'); + ln := regexp_replace(ln, 'Loop Min Rows: \d+ Max Rows: \d+ Total Rows: \d+', + 'Loop Min Rows: N Max Rows: N Total Rows: N'); + return next ln; + end loop; +end; +$$; + prepare ab_q4 (int, int) as select avg(a) from ab where a between $1 and $2 and b < 4; @@ -528,6 +547,12 @@ insert into lprt_a values(3),(3); select explain_parallel_append('select avg(ab.a) from ab inner join lprt_a a on ab.a = a.a where a.a in(1, 0, 3)'); select explain_parallel_append('select avg(ab.a) from ab inner join lprt_a a on ab.a = a.a where a.a in(1, 0, 0)'); +-- Tests for extra statistics +create table lprt_b (b int not null); +insert into lprt_b select generate_series(1,20); +select explain_verbose_parallel_append('select * from lprt_a join lprt_b on a != b'); +drop table lprt_b; + delete from lprt_a where a = 1; select explain_parallel_append('select avg(ab.a) from ab inner join lprt_a a on ab.a = a.a where a.a in(1, 0, 0)'); @@ -654,6 +679,13 @@ select tbl1.col1, tprt.col1 from tbl1 inner join tprt on tbl1.col1 = tprt.col1 order by tbl1.col1, tprt.col1; +-- Tests for extra statistics +explain (analyze, verbose, costs off, summary off, timing off) +select * from tbl1 inner join tprt on tbl1.col1 > tprt.col1; + +explain (analyze, verbose, costs off, summary off, timing off) +select * from tbl1 inner join tprt on tbl1.col1 = tprt.col1; + -- Last partition delete from tbl1; insert into tbl1 values (4400); -- 2.17.1 ^ permalink raw reply [nested|flat] 2+ messages in thread
end of thread, other threads:[~2022-04-02 12:38 UTC | newest] Thread overview: 2+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-11 15:18 [PATCH 5/8] f!2nd attempt to use pkgconfig to allow compiling on OSX Justin Pryzby <[email protected]> 2022-04-02 12:38 Re: [PATCH] Add extra statistics to explain for Nested Loop Justin Pryzby <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox